diff --git a/.env_template b/.env_template index 51392d90..6afc3010 100644 --- a/.env_template +++ b/.env_template @@ -13,4 +13,4 @@ COMPOSE_FILE=docker/default.yml:docker/local.override.yml #COMPOSE_FILE=docker/default.yml:docker/local.override.yml:docker/elasticsearch.yml # If you want to run a specific version, populate this -# REACT_APP_INTELOWL_VERSION="3.0.1" +# REACT_APP_INTELOWL_VERSION="3.1.0" diff --git a/.github/actions/python_requirements/restore_virtualenv/action.yml b/.github/actions/python_requirements/restore_virtualenv/action.yml index cd76c98e..6a68cf71 100644 --- a/.github/actions/python_requirements/restore_virtualenv/action.yml +++ b/.github/actions/python_requirements/restore_virtualenv/action.yml @@ -12,6 +12,9 @@ inputs: description: A git reference (name of the branch, reference to the PR) that will be used to build the cache key. required: false default: ${{ github.ref_name }} + python_version: + description: Python version string to include in the cache key, preventing cross-version cache collisions. + required: true outputs: cache-hit: @@ -32,7 +35,7 @@ runs: uses: actions/cache/restore@v4 with: path: ${{ inputs.virtual_environment_path }} - key: ${{ inputs.git_reference }}-venv-${{ steps.compute_requirements_files_sha256_hash.outputs.computed_hash }} + key: ${{ inputs.git_reference }}-py${{ inputs.python_version }}-venv-${{ steps.compute_requirements_files_sha256_hash.outputs.computed_hash }} - name: Activate restored virtual environment if: > diff --git a/.github/actions/python_requirements/save_virtualenv/action.yml b/.github/actions/python_requirements/save_virtualenv/action.yml index 6c6c66c1..7ec3bcb0 100644 --- a/.github/actions/python_requirements/save_virtualenv/action.yml +++ b/.github/actions/python_requirements/save_virtualenv/action.yml @@ -12,6 +12,9 @@ inputs: description: A git reference (name of the branch, reference to the PR) that will be used to build the cache key. required: false default: ${{ github.ref_name }} + python_version: + description: Python version string to include in the cache key, preventing cross-version cache collisions. + required: true runs: using: "composite" @@ -26,4 +29,4 @@ runs: uses: actions/cache/save@v4 with: path: ${{ inputs.virtual_environment_path }} - key: ${{ inputs.git_reference }}-venv-${{ steps.compute_requirements_files_sha256_hash.outputs.computed_hash }} \ No newline at end of file + key: ${{ inputs.git_reference }}-py${{ inputs.python_version }}-venv-${{ steps.compute_requirements_files_sha256_hash.outputs.computed_hash }} \ No newline at end of file diff --git a/.github/workflows/_python.yml b/.github/workflows/_python.yml index 28fe71bb..e1b0221f 100644 --- a/.github/workflows/_python.yml +++ b/.github/workflows/_python.yml @@ -261,6 +261,7 @@ jobs: uses: actions/checkout@v4 - name: Set up Python + id: setup_python uses: actions/setup-python@v5 with: python-version: ${{ matrix.python_version }} @@ -343,6 +344,7 @@ jobs: uses: ./.github/actions/python_requirements/restore_virtualenv/ with: requirements_paths: "${{ inputs.requirements_path }} requirements-linters.txt requirements-dev.txt requirements-docs.txt" + python_version: ${{ steps.setup_python.outputs.python-version }} - name: Restore Python virtual environment related to target branch id: restore_python_virtual_environment_target_branch @@ -351,6 +353,7 @@ jobs: with: requirements_paths: ${{ inputs.requirements_path }} git_reference: ${{ github.base_ref }} + python_version: ${{ steps.setup_python.outputs.python-version }} - name: Create Python virtual environment if: > @@ -423,6 +426,7 @@ jobs: uses: ./.github/actions/python_requirements/save_virtualenv with: requirements_paths: "${{ inputs.requirements_path }} requirements-linters.txt requirements-dev.txt requirements-docs.txt" + python_version: ${{ steps.setup_python.outputs.python-version }} - name: Save pip cache related to PR event if: > diff --git a/.github/workflows/create_python_cache.yaml b/.github/workflows/create_python_cache.yaml index 8db85f48..15e54143 100644 --- a/.github/workflows/create_python_cache.yaml +++ b/.github/workflows/create_python_cache.yaml @@ -33,9 +33,10 @@ jobs: # sudo apt-get update && sudo apt install ... - name: Set up Python + id: setup_python uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: "3.13" - name: Set up Python virtual environment uses: ./.github/actions/python_requirements/create_virtualenv @@ -52,4 +53,5 @@ jobs: uses: ./.github/actions/python_requirements/save_virtualenv with: requirements_paths: .github/test/python_test/requirements.txt + python_version: ${{ steps.setup_python.outputs.python-version }} diff --git a/.gitignore b/.gitignore index 3d05fec9..61a7b329 100644 --- a/.gitignore +++ b/.gitignore @@ -2,9 +2,14 @@ venv/ docs/build docker/env_file docker/env_file_postgres +docker/env_file.backup.* +docker/env_file_postgres.backup.* .env +.env.backup.* __pycache__/ mlmodels/ +# Backups created by gbctl +backups/ # JetBrains IDEs (PyCharm, IntelliJ, etc.) .idea/ # Ruff cache @@ -16,3 +21,5 @@ coverage.xml *.cover .coverage.* +# gbctl configuration file +.gbctl.conf diff --git a/api/urls.py b/api/urls.py index f426151e..280b8b97 100644 --- a/api/urls.py +++ b/api/urls.py @@ -13,6 +13,7 @@ feeds_asn, feeds_pagination, general_honeypot_list, + news_view, ) # Routers provide an easy way of automatically determining the URL conf. @@ -29,6 +30,7 @@ path("cowrie_session", cowrie_session_view), path("command_sequence", command_sequence_view), path("general_honeypot", general_honeypot_list), + path("news/", news_view), # router viewsets path("", include(router.urls)), # certego_saas: diff --git a/api/views/__init__.py b/api/views/__init__.py index d76a0ef3..73273f93 100644 --- a/api/views/__init__.py +++ b/api/views/__init__.py @@ -3,4 +3,5 @@ from api.views.enrichment import * from api.views.feeds import * from api.views.general_honeypot import * +from api.views.news import * from api.views.statistics import * diff --git a/api/views/news.py b/api/views/news.py new file mode 100644 index 00000000..4800c383 --- /dev/null +++ b/api/views/news.py @@ -0,0 +1,20 @@ +from rest_framework.decorators import api_view +from rest_framework.response import Response + +from api.views.utils import get_greedybear_news + + +@api_view(["GET"]) +def news_view(request): + """ + Fetch GreedyBear blog posts from an RSS feed. + + Filters for posts with "GreedyBear" in the title, truncates long summaries, + sorts by newest first, and caches results to improve performance. + + Returns: + List[dict]: Each dict contains title, date, link, and subtext. + Returns an empty list if no relevant posts are found or feed fails. + """ + news_list = get_greedybear_news() + return Response(news_list) diff --git a/api/views/utils.py b/api/views/utils.py index bc4742c8..f7c2b180 100644 --- a/api/views/utils.py +++ b/api/views/utils.py @@ -6,14 +6,18 @@ from datetime import datetime, timedelta from ipaddress import ip_address +import feedparser +import requests from django.conf import settings from django.contrib.postgres.aggregates import ArrayAgg +from django.core.cache import cache from django.db.models import Count, F, Max, Min, Sum from django.http import HttpResponse, HttpResponseBadRequest, StreamingHttpResponse from rest_framework import status from rest_framework.response import Response from api.serializers import FeedsRequestSerializer +from greedybear.consts import CACHE_KEY_GREEDYBEAR_NEWS, CACHE_TIMEOUT_SECONDS, RSS_FEED_URL from greedybear.models import IOC, GeneralHoneypot, Statistics logger = logging.getLogger(__name__) @@ -401,3 +405,57 @@ def asn_aggregated_queryset(iocs_qs, request, feed_params): result.append(row_dict) return result + + +def get_greedybear_news() -> list[dict]: + """ + Fetch GreedyBear-related blog posts from the IntelOwl RSS feed. + + Returns: + List of dicts with keys: title, date, link, subtext + Sorted newest first, or empty list on failure. + """ + cached = cache.get(CACHE_KEY_GREEDYBEAR_NEWS) + if cached is not None: + return cached + + try: + response = requests.get(RSS_FEED_URL, timeout=5) + response.raise_for_status() + feed = feedparser.parse(response.content) + + filtered_entries = sorted( + [entry for entry in feed.entries if "greedybear" in entry.get("title", "").lower() and entry.get("published_parsed")], + key=lambda e: e.published_parsed, + reverse=True, + ) + + news_items: list[dict] = [] + for entry in filtered_entries: + summary = entry.get("summary", "").strip().replace("\n", " ") + + subtext = summary[:180].rsplit(" ", 1)[0] + "..." if len(summary) > 180 else summary + + news_items.append( + { + "title": entry.get("title"), + "date": entry.get("published"), + "link": entry.get("link"), + "subtext": subtext, + } + ) + + cache.set( + CACHE_KEY_GREEDYBEAR_NEWS, + news_items, + CACHE_TIMEOUT_SECONDS, + ) + + return news_items + + except Exception as exc: + logger.error( + "Failed to fetch GreedyBear news from RSS feed", + exc_info=exc, + ) + return [] diff --git a/configuration/nginx/errors.conf b/configuration/nginx/errors.conf index 7ca2c20c..aceaa520 100644 --- a/configuration/nginx/errors.conf +++ b/configuration/nginx/errors.conf @@ -40,7 +40,7 @@ location /404.json { error_page 408 /408.json; location /408.json { - return 408 '{"code":408,"message":"Request Timeout}'; + return 408 '{"code":408,"message":"Request Timeout"}'; } error_page 413 /413.json; diff --git a/docker/.version b/docker/.version index eac435e2..65d9406b 100644 --- a/docker/.version +++ b/docker/.version @@ -1 +1 @@ -REACT_APP_GREEDYBEAR_VERSION="3.0.1" \ No newline at end of file +REACT_APP_GREEDYBEAR_VERSION="3.1.0" \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index c98f60c2..51e448d2 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,18 +2,17 @@ FROM node:lts-alpine3.22 AS frontend-build WORKDIR / -# copy react source code -COPY frontend/ . -# copy version file as an env file -COPY docker/.version .env.local -# install and build -RUN npm install npm@latest --location=global +# install dependencies first for layer caching +COPY frontend/package.json frontend/package-lock.json ./ RUN npm install +# copy source code and build +COPY frontend/ . +COPY docker/.version .env.local RUN PUBLIC_URL=/static/reactapp/ npm run build # Stage 2: Backend -FROM python:3.13-alpine3.22 +FROM python:3.13-slim-trixie ENV PYTHONUNBUFFERED=1 ENV DJANGO_SETTINGS_MODULE=greedybear.settings @@ -22,12 +21,15 @@ ENV LOG_PATH=/var/log/greedybear ARG WATCHMAN=false -RUN mkdir -p ${LOG_PATH} \ - ${LOG_PATH}/django \ - ${LOG_PATH}/uwsgi \ - # py3-psycopg2 is required to use PostgresSQL with Django \ - # libgomp is required to train the model - && apk --no-cache -U add bash py3-psycopg2 gcc python3-dev alpine-sdk linux-headers libgomp \ +RUN mkdir -p ${LOG_PATH}/django ${LOG_PATH}/uwsgi \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + # Build dependencies (removed after pip install) + gcc python3-dev \ + # Runtime dependencies: + # libexpat1 is required by uWSGI/Python, + # libgomp1 is required for model training, netcat-openbsd for healthcheck + libexpat1 libgomp1 netcat-openbsd \ && pip3 install --no-cache-dir --upgrade pip COPY requirements/project-requirements.txt $PYTHONPATH/project-requirements.txt @@ -51,11 +53,13 @@ RUN touch ${LOG_PATH}/django/api.log ${LOG_PATH}/django/api_errors.log \ && touch ${LOG_PATH}/django/django_errors.log ${LOG_PATH}/django/elasticsearch.log\ && touch ${LOG_PATH}/django/authentication.log ${LOG_PATH}/django/authentication_errors.log \ && mkdir -p ${PYTHONPATH}/mlmodels \ - && adduser -S -H -u 2000 -D -g www-data www-data \ + && usermod -u 2000 www-data \ && chown -R www-data:www-data ${LOG_PATH} /opt/deploy/ ${PYTHONPATH}/mlmodels/ \ && rm -rf docs/ frontend/ tests/ .github/ docker/hooks/ \ && /bin/bash ./docker/watchman_install.sh \ - && apk del gcc python3-dev alpine-sdk linux-headers + && apt-get purge -y gcc python3-dev \ + && apt-get autoremove -y \ + && rm -rf /var/lib/apt/lists/* # remove the apt package index cache # start period is high to allow data migration for 1.4.0 HEALTHCHECK --interval=10s --timeout=2s --start-period=500s --retries=3 CMD nc -z localhost 8001 || exit 1 diff --git a/docker/Dockerfile_nginx b/docker/Dockerfile_nginx index 049ea8c9..dc8a3172 100644 --- a/docker/Dockerfile_nginx +++ b/docker/Dockerfile_nginx @@ -1,4 +1,4 @@ -FROM library/nginx:1.29.4-alpine +FROM library/nginx:1.29.5-alpine RUN mkdir -p /var/cache/nginx /var/cache/nginx/feeds RUN apk update && apk upgrade && apk add bash ENV NGINX_LOG_DIR=/var/log/nginx diff --git a/docker/entrypoint_uwsgi.sh b/docker/entrypoint_uwsgi.sh index ed995ca0..ddbe7b87 100755 --- a/docker/entrypoint_uwsgi.sh +++ b/docker/entrypoint_uwsgi.sh @@ -15,10 +15,15 @@ echo "Waiting for db to be ready..." python manage.py makemigrations durin python manage.py migrate -# Collect static files -python manage.py collectstatic --noinput +# Collect static files, overwriting existing ones +python manage.py collectstatic --noinput --clear --verbosity 0 + +# Obtain the current GreedyBear version number +. /opt/deploy/greedybear/docker/.version +export REACT_APP_GREEDYBEAR_VERSION echo "------------------------------" +echo "GreedyBear $REACT_APP_GREEDYBEAR_VERSION" echo "DEBUG: " $DEBUG echo "DJANGO_TEST_SERVER: " $DJANGO_TEST_SERVER echo "------------------------------" diff --git a/docker/watchman_install.sh b/docker/watchman_install.sh index e9a63734..250b945d 100755 --- a/docker/watchman_install.sh +++ b/docker/watchman_install.sh @@ -3,7 +3,7 @@ # This script can be disabled during development using REPO_DOWNLOADER_ENABLED=true env variable if [ "$WATCHMAN" = "false" ]; then echo "Skipping WATCHMAN installation because we are not in test mode"; exit 0; fi -apk add gcc linux-headers build-base +apt-get update && apt-get install -y --no-install-recommends gcc build-essential pip3 install --compile -r requirements/django-server-requirements.txt # install Watchman to enhance performance on the Django development Server diff --git a/frontend/package-lock.json b/frontend/package-lock.json index e75372ca..cf18545a 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -9,7 +9,7 @@ "version": "0.1.0", "dependencies": { "@certego/certego-ui": "^0.1.10", - "axios": "^1.12.0", + "axios": "^1.13.5", "axios-hooks": "^3.0.4", "bootstrap": ">=5.3.0", "formik": "^2.2.9", @@ -5620,12 +5620,13 @@ } }, "node_modules/axios": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.2.tgz", - "integrity": "sha512-7i24Ri4pmDRfJTR7LDBhsOTtcm+9kjX5WiY1X3wIisx6G9So3pfMkEiU7emUBe46oceVImccTEM3k6C5dbVW8A==", + "version": "1.13.5", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.13.5.tgz", + "integrity": "sha512-cz4ur7Vb0xS4/KUN0tPWe44eqxrIu31me+fbang3ijiNscE129POzipJJA6zniq2C/Z6sJCjMimjS8Lc/GAs8Q==", + "license": "MIT", "dependencies": { - "follow-redirects": "^1.15.0", - "form-data": "^4.0.0", + "follow-redirects": "^1.15.11", + "form-data": "^4.0.5", "proxy-from-env": "^1.1.0" } }, @@ -6149,6 +6150,19 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -7688,6 +7702,20 @@ "resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-5.1.0.tgz", "integrity": "sha512-YXQl1DSa4/PQyRfgrv6aoNjhasp/p4qs9FjJ4q4cQk+8m4r6k4ZSiEyytKG8f8W9gi8WsQtIObNmKd+tMzNTmA==" }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/duplexer": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz", @@ -7846,6 +7874,24 @@ "resolved": "https://registry.npmjs.org/es-array-method-boxes-properly/-/es-array-method-boxes-properly-1.0.0.tgz", "integrity": "sha512-wd6JXUmyHmt8T5a2xreUwKcGPq6f1f+WwIJkijUqiGcJz1qqnZgP6XIK+QyIWU5lT7imeNxUll48bziG+TSYcA==" }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/es-get-iterator": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/es-get-iterator/-/es-get-iterator-1.1.3.tgz", @@ -7875,14 +7921,28 @@ "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-0.9.3.tgz", "integrity": "sha512-1HQ2M2sPtxwnvOvT1ZClHyQDiggdNjURWpY2we6aMKCQiUVxTmVs2UYPLIrD84sS+kMdUwfBSylbJPwNnBrnHQ==" }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/es-set-tostringtag": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.0.1.tgz", - "integrity": "sha512-g3OMbtlwY3QewlqAiMLI47KywjWZoEytKr8pf6iTC8uJq5bIAH52Z9pnQ8pVL6whrCto53JZDuUIsifGeLorTg==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", + "license": "MIT", "dependencies": { - "get-intrinsic": "^1.1.3", - "has": "^1.0.3", - "has-tostringtag": "^1.0.0" + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" }, "engines": { "node": ">= 0.4" @@ -9099,15 +9159,16 @@ } }, "node_modules/follow-redirects": { - "version": "1.15.6", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", - "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", + "version": "1.15.11", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz", + "integrity": "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==", "funding": [ { "type": "individual", "url": "https://github.com/sponsors/RubenVerborgh" } ], + "license": "MIT", "engines": { "node": ">=4.0" }, @@ -9226,12 +9287,15 @@ } }, "node_modules/form-data": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", - "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", + "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==", + "license": "MIT", "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.2", "mime-types": "^2.1.12" }, "engines": { @@ -9326,9 +9390,13 @@ } }, "node_modules/function-bind": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } }, "node_modules/function.prototype.name": { "version": "1.1.5", @@ -9377,13 +9445,24 @@ } }, "node_modules/get-intrinsic": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.0.tgz", - "integrity": "sha512-L049y6nFOuom5wGyRc3/gdTLO94dySVKRACj1RmJZBQXlbTMhtNIgkWkUHq+jYmZvKf14EW1EoJnnjbmoHij0Q==", - "dependencies": { - "function-bind": "^1.1.1", - "has": "^1.0.3", - "has-symbols": "^1.0.3" + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -9402,6 +9481,19 @@ "node": ">=8.0.0" } }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/get-stream": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", @@ -9546,11 +9638,12 @@ "dev": true }, "node_modules/gopd": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", - "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", - "dependencies": { - "get-intrinsic": "^1.1.3" + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -9635,9 +9728,10 @@ } }, "node_modules/has-symbols": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", - "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", "engines": { "node": ">= 0.4" }, @@ -9646,11 +9740,12 @@ } }, "node_modules/has-tostringtag": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz", - "integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", + "license": "MIT", "dependencies": { - "has-symbols": "^1.0.2" + "has-symbols": "^1.0.3" }, "engines": { "node": ">= 0.4" @@ -9659,6 +9754,18 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/he": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/he/-/he-1.2.0.tgz", @@ -13252,6 +13359,15 @@ "remove-accents": "0.4.2" } }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, "node_modules/mathml-tag-names": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/mathml-tag-names/-/mathml-tag-names-2.1.3.tgz", @@ -24893,12 +25009,12 @@ "integrity": "sha512-/BQzOX780JhsxDnPpH4ZiyrJAzcd8AfzFPkv+89veFSr1rcMjuq2JDCwypKaPeB6ljHp9KjXhPpjgCvQlWYuqg==" }, "axios": { - "version": "1.6.2", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.2.tgz", - "integrity": "sha512-7i24Ri4pmDRfJTR7LDBhsOTtcm+9kjX5WiY1X3wIisx6G9So3pfMkEiU7emUBe46oceVImccTEM3k6C5dbVW8A==", + "version": "1.13.5", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.13.5.tgz", + "integrity": "sha512-cz4ur7Vb0xS4/KUN0tPWe44eqxrIu31me+fbang3ijiNscE129POzipJJA6zniq2C/Z6sJCjMimjS8Lc/GAs8Q==", "requires": { - "follow-redirects": "^1.15.0", - "form-data": "^4.0.0", + "follow-redirects": "^1.15.11", + "form-data": "^4.0.5", "proxy-from-env": "^1.1.0" } }, @@ -25296,6 +25412,15 @@ "get-intrinsic": "^1.0.2" } }, + "call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "requires": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + } + }, "callsites": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/callsites/-/callsites-3.1.0.tgz", @@ -26421,6 +26546,16 @@ "resolved": "https://registry.npmjs.org/dotenv-expand/-/dotenv-expand-5.1.0.tgz", "integrity": "sha512-YXQl1DSa4/PQyRfgrv6aoNjhasp/p4qs9FjJ4q4cQk+8m4r6k4ZSiEyytKG8f8W9gi8WsQtIObNmKd+tMzNTmA==" }, + "dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "requires": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + } + }, "duplexer": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/duplexer/-/duplexer-0.1.2.tgz", @@ -26546,6 +26681,16 @@ "resolved": "https://registry.npmjs.org/es-array-method-boxes-properly/-/es-array-method-boxes-properly-1.0.0.tgz", "integrity": "sha512-wd6JXUmyHmt8T5a2xreUwKcGPq6f1f+WwIJkijUqiGcJz1qqnZgP6XIK+QyIWU5lT7imeNxUll48bziG+TSYcA==" }, + "es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==" + }, + "es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==" + }, "es-get-iterator": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/es-get-iterator/-/es-get-iterator-1.1.3.tgz", @@ -26574,14 +26719,23 @@ "resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-0.9.3.tgz", "integrity": "sha512-1HQ2M2sPtxwnvOvT1ZClHyQDiggdNjURWpY2we6aMKCQiUVxTmVs2UYPLIrD84sS+kMdUwfBSylbJPwNnBrnHQ==" }, + "es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "requires": { + "es-errors": "^1.3.0" + } + }, "es-set-tostringtag": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.0.1.tgz", - "integrity": "sha512-g3OMbtlwY3QewlqAiMLI47KywjWZoEytKr8pf6iTC8uJq5bIAH52Z9pnQ8pVL6whrCto53JZDuUIsifGeLorTg==", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", + "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", "requires": { - "get-intrinsic": "^1.1.3", - "has": "^1.0.3", - "has-tostringtag": "^1.0.0" + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.6", + "has-tostringtag": "^1.0.2", + "hasown": "^2.0.2" } }, "es-shim-unscopables": { @@ -27489,9 +27643,9 @@ } }, "follow-redirects": { - "version": "1.15.6", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", - "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==" + "version": "1.15.11", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz", + "integrity": "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==" }, "for-each": { "version": "0.3.3", @@ -27567,12 +27721,14 @@ } }, "form-data": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", - "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz", + "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==", "requires": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", + "es-set-tostringtag": "^2.1.0", + "hasown": "^2.0.2", "mime-types": "^2.1.12" } }, @@ -27632,9 +27788,9 @@ "optional": true }, "function-bind": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==" }, "function.prototype.name": { "version": "1.1.5", @@ -27668,13 +27824,20 @@ "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" }, "get-intrinsic": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.0.tgz", - "integrity": "sha512-L049y6nFOuom5wGyRc3/gdTLO94dySVKRACj1RmJZBQXlbTMhtNIgkWkUHq+jYmZvKf14EW1EoJnnjbmoHij0Q==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", "requires": { - "function-bind": "^1.1.1", - "has": "^1.0.3", - "has-symbols": "^1.0.3" + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" } }, "get-own-enumerable-property-symbols": { @@ -27687,6 +27850,15 @@ "resolved": "https://registry.npmjs.org/get-package-type/-/get-package-type-0.1.0.tgz", "integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==" }, + "get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "requires": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + } + }, "get-stream": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz", @@ -27788,12 +27960,9 @@ "dev": true }, "gopd": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", - "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", - "requires": { - "get-intrinsic": "^1.1.3" - } + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==" }, "graceful-fs": { "version": "4.2.10", @@ -27850,16 +28019,24 @@ "integrity": "sha512-7qE+iP+O+bgF9clE5+UoBFzE65mlBiVj3tKCrlNQ0Ogwm0BjpT/gK4SlLYDMybDh5I3TCTKnPPa0oMG7JDYrhg==" }, "has-symbols": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", - "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==" + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==" }, "has-tostringtag": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.0.tgz", - "integrity": "sha512-kFjcSNhnlGV1kyoGk7OXKSawH5JOb/LzUc5w9B02hOTO0dfFRjbHQKvg1d6cf3HbeUmtU9VbbV3qzZ2Teh97WQ==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", + "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", "requires": { - "has-symbols": "^1.0.2" + "has-symbols": "^1.0.3" + } + }, + "hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "requires": { + "function-bind": "^1.1.2" } }, "he": { @@ -30677,6 +30854,11 @@ "remove-accents": "0.4.2" } }, + "math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==" + }, "mathml-tag-names": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/mathml-tag-names/-/mathml-tag-names-2.1.3.tgz", diff --git a/frontend/package.json b/frontend/package.json index f4c9d883..e59bb3ef 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -10,7 +10,7 @@ }, "dependencies": { "@certego/certego-ui": "^0.1.10", - "axios": "^1.6.0", + "axios": "^1.13.5", "axios-hooks": "^3.0.4", "bootstrap": ">=5.3.0", "formik": "^2.2.9", diff --git a/frontend/src/components/feeds/tableColumns.jsx b/frontend/src/components/feeds/tableColumns.jsx index 804b473a..ed118e22 100644 --- a/frontend/src/components/feeds/tableColumns.jsx +++ b/frontend/src/components/feeds/tableColumns.jsx @@ -1,6 +1,19 @@ -import { BooleanIcon } from "@certego/certego-ui"; +import { UncontrolledPopover, PopoverBody } from "reactstrap"; +import { FiInfo } from "react-icons/fi"; +import { BooleanIcon, IconButton } from "@certego/certego-ui"; -// costants +const formatInteger = (value) => { + if (value === null || value === undefined || Number.isNaN(value)) return "-"; + return Number(value).toLocaleString(); +}; + +// required for recurrence value +const formatPercent = (value) => { + if (value === null || value === undefined || Number.isNaN(value)) return "-"; + return `${(Number(value) * 100).toFixed(1)}%`; +}; + +// constants const feedsTableColumns = [ { Header: "Last Seen", @@ -44,13 +57,70 @@ const feedsTableColumns = [ Header: "Payload Request", accessor: "payload_request", Cell: ({ value }) => , - maxWidth: 70, + maxWidth: 60, }, { Header: "Attack Count", accessor: "attack_count", maxWidth: 60, }, + { + Header: "Details", + accessor: "details", + Cell: ({ row }) => { + const { + recurrence_probability, + expected_interactions, + interaction_count, + destination_port_count, + login_attempts, + asn, + ip_reputation, + } = row.original; + const popoverId = `feeds-details-${row.id}`; + return ( +
+ + + +
Scores
+
Recurrence: {formatPercent(recurrence_probability)}
+
+ Expected Interactions:{" "} + {formatInteger( + expected_interactions != null + ? Math.round(expected_interactions) + : null + )} +
+
+
Activity
+
Interactions: {formatInteger(interaction_count)}
+
Ports: {formatInteger(destination_port_count)}
+
Logins: {formatInteger(login_attempts)}
+
+
Enrichment
+
ASN: {asn ?? "-"}
+
Reputation: {ip_reputation || "-"}
+ + +
+ ); + }, + maxWidth: 60, + }, ]; export { feedsTableColumns }; diff --git a/frontend/src/components/home/Home.jsx b/frontend/src/components/home/Home.jsx index 8ea6cf00..9354f214 100644 --- a/frontend/src/components/home/Home.jsx +++ b/frontend/src/components/home/Home.jsx @@ -6,36 +6,11 @@ import { Container } from "reactstrap"; import { ContentSection } from "@certego/certego-ui"; import { PUBLIC_URL, VERSION } from "../../constants/environment"; +import { NewsWidget } from "./NewsWidget"; const versionText = VERSION; // const versionText = "v1.0.0"; const logoBgImg = `url('${PUBLIC_URL}/greedybear.png')`; -const blogPosts = [ - { - title: "GreedyBear version 3.0 coming", - subText: "With many new features!", - date: "29th January 2026", - link: "https://intelowlproject.github.io/blogs/greedybear_v3_release", - }, - { - title: "GreedyBear version 2.0 released", - subText: "Upgrade from 1.x requires manual intervention", - date: "3rd October 2025", - link: "https://intelowlproject.github.io/blogs/greedybear_v2_release", - }, - { - title: "Improvements to GreedyBear", - subText: "Machine Learning applied to GreedyBear Feeds", - date: "28th May 2025", - link: "https://intelowlproject.github.io/blogs/improvements_to_greedybear", - }, - { - title: "New project available: GreedyBear", - subText: "Honeynet Blog: Official announcement", - date: "27th December 2021", - link: "https://www.honeynet.org/2021/12/27/new-project-available-greedybear/", - }, -]; function Home() { console.debug("Home rendered!"); @@ -60,24 +35,10 @@ function Home() { prevent and detect attacks.
- {/* blogPosts */} +
GreedyBear News
- {blogPosts.map(({ title, subText, date, link }) => ( - - {date} -
{title}
-

{subText}

- - Read - - - ))} + diff --git a/frontend/src/components/home/NewsWidget.jsx b/frontend/src/components/home/NewsWidget.jsx new file mode 100644 index 00000000..60957f62 --- /dev/null +++ b/frontend/src/components/home/NewsWidget.jsx @@ -0,0 +1,97 @@ +import React from "react"; +import { ContentSection } from "@certego/certego-ui"; +import { Spinner } from "reactstrap"; +import { GREEDYBEAR_NEWS_URL } from "../../constants/api"; + +export const NewsWidget = React.memo(() => { + const [data, setData] = React.useState([]); + const [loading, setLoading] = React.useState(true); + const [error, setError] = React.useState(false); + + const formatDate = (dateStr) => { + if (!dateStr) return ""; + + const date = new Date(dateStr); + if (isNaN(date.getTime())) return dateStr; + + const day = date.getDate(); + const month = date.toLocaleDateString("en-US", { month: "short" }); + const year = date.getFullYear(); + + const ordinals = ["th", "st", "nd", "rd"]; + const v = day % 100; + const ordinal = ordinals[(v - 20) % 10] || ordinals[v] || ordinals[0]; + + return `${day}${ordinal} ${month} ${year}`; + }; + + React.useEffect(() => { + fetch(GREEDYBEAR_NEWS_URL) + .then((response) => { + if (!response.ok) { + throw new Error(`HTTP error ${response.status}`); + } + return response.json(); + }) + .then(setData) + .catch((err) => { + console.error("Error fetching news:", err); + setError(true); + }) + .finally(() => setLoading(false)); + }, []); + + if (loading) { + return ( +
+ + Loading news... +
+ ); + } + + if (error) { + return ( +
+ + Unable to load news. Please try again later. + +
+ ); + } + + if (data.length === 0) { + return ( +
+ No news available at the moment. +
+ ); + } + + return ( + <> + {data.map((item) => ( + + {item.date && ( + + {formatDate(item.date)} + + )} + +
{item.title}
+

{item.subtext}

+ + + Read more + + + ))} + + ); +}); diff --git a/frontend/src/constants/api.js b/frontend/src/constants/api.js index 568f034a..016acaed 100644 --- a/frontend/src/constants/api.js +++ b/frontend/src/constants/api.js @@ -23,3 +23,6 @@ export const FEEDS_BASE_URI = `${API_BASE_URI}/feeds`; //honeypot export const GENERAL_HONEYPOT_URI = `${API_BASE_URI}/general_honeypot`; + +// News +export const GREEDYBEAR_NEWS_URL = `${API_BASE_URI}/news`; diff --git a/frontend/src/styles/App.scss b/frontend/src/styles/App.scss index 045d47b4..29762562 100644 --- a/frontend/src/styles/App.scss +++ b/frontend/src/styles/App.scss @@ -152,3 +152,24 @@ section.fixed-bottom { .recharts-tooltip-wrapper { z-index: 99999 !important; } + +.feeds-details-popover { + .popover { + background-color: $darker; + } + + .popover-body { + color: $light; + } + + .popover-header { + background-color: $darker; + color: $light; + border-bottom: 1px solid rgba(255, 255, 255, 0.08); + } + + .popover-arrow::before, + .popover-arrow::after { + border-color: $darker; + } +} diff --git a/frontend/tests/components/feeds/TableColumns.test.jsx b/frontend/tests/components/feeds/TableColumns.test.jsx new file mode 100644 index 00000000..69f80a96 --- /dev/null +++ b/frontend/tests/components/feeds/TableColumns.test.jsx @@ -0,0 +1,45 @@ +import React from "react"; +import "@testing-library/jest-dom"; +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { feedsTableColumns } from "../../../src/components/feeds/tableColumns"; + +describe("Feeds table details popover", () => { + test("shows details button and popover content on click", async () => { + const user = userEvent.setup(); + const detailsColumn = feedsTableColumns.find( + (column) => column.accessor === "details" + ); + expect(detailsColumn).toBeDefined(); + + const row = { + id: "0", + original: { + recurrence_probability: 0.25, + expected_interactions: 12.7, + interaction_count: 10, + destination_port_count: 2, + login_attempts: 3, + asn: "AS123", + ip_reputation: "benign", + }, + }; + + const DetailsCell = detailsColumn.Cell; + + render(); + + const detailsButton = screen.getByLabelText(/view details/i); + expect(detailsButton).toBeInTheDocument(); + expect(screen.queryByText(/Recurrence:/i)).not.toBeInTheDocument(); + + await user.click(detailsButton); + + expect( + await screen.findByText(/Recurrence:\s*25\.0%/i) + ).toBeInTheDocument(); + expect( + await screen.findByText(/Expected Interactions:\s*13/i) + ).toBeInTheDocument(); + }); +}); diff --git a/frontend/tests/components/home/NewsWidget.test.jsx b/frontend/tests/components/home/NewsWidget.test.jsx new file mode 100644 index 00000000..b6f3355d --- /dev/null +++ b/frontend/tests/components/home/NewsWidget.test.jsx @@ -0,0 +1,257 @@ +import React from "react"; +import { render, screen, waitFor } from "@testing-library/react"; +import { NewsWidget } from "../../../src/components/home/NewsWidget"; +import { GREEDYBEAR_NEWS_URL } from "../../../src/constants/api"; +import "@testing-library/jest-dom"; + +global.fetch = jest.fn(); + +describe("NewsWidget", () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + afterEach(() => { + jest.resetAllMocks(); + }); + + describe("Loading State", () => { + it("should display loading spinner while fetching data", () => { + fetch.mockImplementation(() => new Promise(() => {})); + + render(); + + expect(screen.getByText("Loading news...")).toBeInTheDocument(); + expect(screen.getByRole("status")).toBeInTheDocument(); + }); + }); + + describe("Success State", () => { + it("should display news items when data is fetched successfully", async () => { + const mockNewsData = [ + { + date: "2024-01-15", + title: "Test News Title 1", + subtext: "Test subtext 1", + link: "https://example.com/news1", + }, + { + date: "2024-02-20", + title: "Test News Title 2", + subtext: "Test subtext 2", + link: "https://example.com/news2", + }, + ]; + + fetch.mockResolvedValueOnce({ + ok: true, + json: async () => mockNewsData, + }); + + render(); + + // Wait for the first title to appear + expect(await screen.findByText("Test News Title 1")).toBeInTheDocument(); + + // Check the rest without waitFor + expect(screen.getByText("Test News Title 2")).toBeInTheDocument(); + expect(screen.getByText("Test subtext 1")).toBeInTheDocument(); + expect(screen.getByText("Test subtext 2")).toBeInTheDocument(); + }); + + it("should call the correct API endpoint", async () => { + fetch.mockResolvedValueOnce({ + ok: true, + json: async () => [], + }); + + render(); + + await waitFor(() => { + expect(fetch).toHaveBeenCalledWith(GREEDYBEAR_NEWS_URL); + }); + expect(fetch).toHaveBeenCalledTimes(1); + }); + + it("should render 'Read more' links with correct href", async () => { + const mockNewsData = [ + { + date: "2024-01-15", + title: "Test News", + subtext: "Test subtext", + link: "https://example.com/news", + }, + ]; + + fetch.mockResolvedValueOnce({ + ok: true, + json: async () => mockNewsData, + }); + + render(); + + const link = await screen.findByText("Read more"); + expect(link).toHaveAttribute("href", "https://example.com/news"); + expect(link).toHaveAttribute("target", "_blank"); + expect(link).toHaveAttribute("rel", "noopener noreferrer"); + }); + + it("should display message when no news is available", async () => { + fetch.mockResolvedValueOnce({ + ok: true, + json: async () => [], + }); + + render(); + + expect( + await screen.findByText("No news available at the moment.") + ).toBeInTheDocument(); + }); + }); + + describe("Error State", () => { + it("should display error message when fetch fails", async () => { + fetch.mockRejectedValueOnce(new Error("Network error")); + + render(); + + expect( + await screen.findByText("Unable to load news. Please try again later.") + ).toBeInTheDocument(); + }); + + it("should display error message when response is not ok", async () => { + fetch.mockResolvedValueOnce({ + ok: false, + status: 404, + }); + + render(); + + expect( + await screen.findByText("Unable to load news. Please try again later.") + ).toBeInTheDocument(); + }); + + it("should log error to console when fetch fails", async () => { + const consoleErrorSpy = jest + .spyOn(console, "error") + .mockImplementation(() => {}); + const mockError = new Error("Network error"); + + fetch.mockRejectedValueOnce(mockError); + + render(); + + await waitFor(() => { + expect(consoleErrorSpy).toHaveBeenCalledWith( + "Error fetching news:", + mockError + ); + }); + + consoleErrorSpy.mockRestore(); + }); + }); + + describe("Date Formatting", () => { + it("should format date correctly with ordinal suffix", async () => { + const mockNewsData = [ + { date: "2024-01-01", title: "N1", subtext: "T", link: "L1" }, + { date: "2024-02-02", title: "N2", subtext: "T", link: "L2" }, + { date: "2024-03-03", title: "N3", subtext: "T", link: "L3" }, + { date: "2024-04-21", title: "N4", subtext: "T", link: "L4" }, + ]; + + fetch.mockResolvedValueOnce({ ok: true, json: async () => mockNewsData }); + + render(); + + expect(await screen.findByText("1st Jan 2024")).toBeInTheDocument(); + expect(screen.getByText("2nd Feb 2024")).toBeInTheDocument(); + expect(screen.getByText("3rd Mar 2024")).toBeInTheDocument(); + expect(screen.getByText("21st Apr 2024")).toBeInTheDocument(); + }); + + it("should handle items without date", async () => { + const mockNewsData = [{ title: "No date", subtext: "T", link: "L" }]; + fetch.mockResolvedValueOnce({ ok: true, json: async () => mockNewsData }); + + render(); + + expect(await screen.findByText("No date")).toBeInTheDocument(); + expect(screen.queryByText(/Jan|Feb|Mar/)).not.toBeInTheDocument(); + }); + + it("should handle invalid date strings gracefully", async () => { + const mockNewsData = [ + { date: "invalid-date", title: "Invalid", subtext: "T", link: "L" }, + ]; + fetch.mockResolvedValueOnce({ ok: true, json: async () => mockNewsData }); + + render(); + + expect(await screen.findByText("Invalid")).toBeInTheDocument(); + expect(screen.getByText("invalid-date")).toBeInTheDocument(); + }); + + it("should format dates with correct ordinal suffix for 11th-13th", async () => { + const mockNewsData = [ + { date: "2024-01-11", title: "11", subtext: "T", link: "L1" }, + { date: "2024-01-12", title: "12", subtext: "T", link: "L2" }, + { date: "2024-01-13", title: "13", subtext: "T", link: "L3" }, + ]; + + fetch.mockResolvedValueOnce({ ok: true, json: async () => mockNewsData }); + + render(); + + expect(await screen.findByText("11th Jan 2024")).toBeInTheDocument(); + expect(screen.getByText("12th Jan 2024")).toBeInTheDocument(); + expect(screen.getByText("13th Jan 2024")).toBeInTheDocument(); + }); + }); + + describe("Component Behavior", () => { + it("should fetch data only once on mount", async () => { + fetch.mockResolvedValueOnce({ ok: true, json: async () => [] }); + const { rerender } = render(); + + await waitFor(() => { + expect(fetch).toHaveBeenCalledTimes(1); + }); + + rerender(); + expect(fetch).toHaveBeenCalledTimes(1); + }); + + it("should render multiple news items correctly", async () => { + const mockNewsData = Array.from({ length: 5 }, (_, i) => ({ + date: `2024-01-${i + 1}`, + title: `News Title ${i + 1}`, + subtext: `Subtext ${i + 1}`, + link: `https://example.com/news${i + 1}`, + })); + + fetch.mockResolvedValueOnce({ ok: true, json: async () => mockNewsData }); + + render(); + + expect(await screen.findByText("News Title 1")).toBeInTheDocument(); + + mockNewsData.forEach((item) => { + expect(screen.getByText(item.title)).toBeInTheDocument(); + expect(screen.getByText(item.subtext)).toBeInTheDocument(); + }); + + expect(screen.getAllByText("Read more")).toHaveLength(5); + }); + }); + + describe("Memoization", () => { + it("should be wrapped with React.memo", () => { + expect(NewsWidget.$$typeof).toBeDefined(); + }); + }); +}); diff --git a/gbctl b/gbctl new file mode 100755 index 00000000..2a959758 --- /dev/null +++ b/gbctl @@ -0,0 +1,1235 @@ +#!/usr/bin/env bash + +# GreedyBear Setup and Management Script +# This script simplifies deployment, updates, and management of GreedyBear instances + +set -euo pipefail + +# Resolve script directory for absolute path usage +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Version and defaults +GREEDYBEAR_VERSION="latest" +PROJECT_NAME="greedybear" +SILENT_MODE=false +ELASTIC_ENDPOINT="" + +# Admin creation defaults +ADMIN_USERNAME="${GB_ADMIN_USERNAME:-}" +ADMIN_PASSWORD="${GB_ADMIN_PASSWORD:-}" +ADMIN_EMAIL="${GB_ADMIN_EMAIL:-}" +# Color codes for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +NC='\033[0m' # No Color + +# Environment and command validation arrays +declare -A cmd_arguments=(["init"]=1 ["up"]=1 ["start"]=1 ["down"]=1 ["stop"]=1 ["restart"]=1 ["logs"]=1 ["ps"]=1 ["update"]=1 ["build"]=1 ["pull"]=1 ["backup"]=1 ["restore"]=1 ["health"]=1 ["clean"]=1 ["create-admin"]=1) + +# Logging functions +log_info() { + if [ "$SILENT_MODE" = false ]; then + echo -e "${BLUE}[INFO]${NC} $1" + fi +} + +log_success() { + if [ "$SILENT_MODE" = false ]; then + echo -e "${GREEN}[SUCCESS]${NC} $1" + fi +} + +log_warning() { + if [ "$SILENT_MODE" = false ]; then + echo -e "${YELLOW}[WARNING]${NC} $1" + fi +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $1" >&2 +} + +# Save configuration to .env file +save_config() { + local env_file="${SCRIPT_DIR}/.env" + log_info "Saving configuration to ${env_file}..." + + # Build COMPOSE_FILE from flags + local compose_file="docker/default.yml" + [ "$ENV_MODE" = "dev" ] && compose_file+=":docker/local.override.yml" + [ "$ENABLE_HTTPS" = true ] && compose_file+=":docker/https.override.yml" + [ "$ENABLE_ELASTIC" = true ] && compose_file+=":docker/elasticsearch.yml" + [ "$USE_VERSION" = true ] && compose_file+=":docker/version.override.yml" + + cat > "$env_file" <> "$env_file" + fi + + log_success "Configuration saved" +} + +# Load configuration from .env file +load_config() { + local env_file="${SCRIPT_DIR}/.env" + if [ -f "$env_file" ]; then + if [ "$SILENT_MODE" = false ] && [ "$COMMAND" != "init" ]; then + log_info "Loading configuration from ${env_file}..." + fi + + local compose_file="" + local version_pin="" + + # Parse KEY=VALUE pairs + while IFS='=' read -r key value; do + [[ -z "$key" || "$key" == \#* ]] && continue + key="${key#"${key%%[![:space:]]*}"}" + key="${key%"${key##*[![:space:]]}"}" + value="${value%\"}" + value="${value#\"}" + case "$key" in + COMPOSE_FILE) compose_file="$value" ;; + REACT_APP_INTELOWL_VERSION) version_pin="$value" ;; + esac + done < "$env_file" + + # Derive gbctl flags from COMPOSE_FILE + if [ -n "$compose_file" ]; then + [[ "$compose_file" == *"local.override.yml"* ]] && ENV_MODE="dev" || ENV_MODE="prod" + [[ "$compose_file" == *"https.override.yml"* ]] && ENABLE_HTTPS=true + [[ "$compose_file" == *"elasticsearch.yml"* ]] && ENABLE_ELASTIC=true + if [[ "$compose_file" == *"version.override.yml"* ]]; then + USE_VERSION=true + GREEDYBEAR_VERSION="${version_pin:-latest}" + fi + fi + fi +} + +# Print help message +print_help() { + cat << EOF +GreedyBear - Automated Setup and Management Script + +SYNOPSIS + ./gbctl [OPTIONS] + ./gbctl -h|--help + +COMMANDS + init Initialize GreedyBear (setup environment files) + up Start all services + start Alias for 'up' + down Stop and remove all services + stop Stop all services without removing them + restart Restart all services + logs View logs from services + ps List running services + update Update GreedyBear to latest version + build Build Docker images + pull Pull latest Docker images + backup Backup database and volumes + restore Restore from backup + health Health check all services + clean Remove all data and reset (destructive!) + create-admin Create a Django superuser (interactive or silent) + +GLOBAL OPTIONS + -h, --help Show this help message + -s, --silent Silent mode (non-interactive) + +INIT OPTIONS (./gbctl init) + --dev Development deployment with hot-reload + --release Pin a specific image tag (e.g. stag, 3.0.1) + --https Enable HTTPS with custom certificates + --elastic-local Enable local Elasticsearch (requires >=16GB RAM) + --elastic-endpoint Specify Elasticsearch endpoint + + Without --dev or --release, production mode with the "prod" image tag is used. + +CLEAN OPTIONS (./gbctl clean) + --force Required to confirm destructive operation + +CREATE-ADMIN OPTIONS (./gbctl create-admin) + --username Username (or use GB_ADMIN_USERNAME env var) + --password Password (or use GB_ADMIN_PASSWORD env var) + --email Email address (or use GB_ADMIN_EMAIL env var) + +LOGS OPTIONS + ./gbctl logs View container logs (stdout/stderr) + ./gbctl logs app View Django application logs inside container + +EXAMPLES + # Initialize production deployment with external TPOT Elasticsearch + ./gbctl init --elastic-endpoint http://tpot-host:64298 + + # Initialize for development with a dummy Elasticsearch instance + ./gbctl init --dev --elastic-local + + # Initialize staging deployment + ./gbctl init --release stag + + # Start services (uses configuration from init) + ./gbctl up + + # View logs + ./gbctl logs + + # Create admin user + ./gbctl create-admin --username admin --password secret --email admin@example.com + + # Update to latest version + ./gbctl update + + # Silent initialization + ./gbctl init --silent + +EOF +} + +# Check if Docker is installed +check_docker() { + log_info "Checking Docker installation..." + + if ! command -v docker &> /dev/null; then + log_error "Docker is not installed." + if [ "$SILENT_MODE" = false ]; then + read -p "Would you like to install Docker? [y/N] " -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]]; then + install_docker + else + log_error "Docker is required to run GreedyBear. Exiting." + exit 1 + fi + else + log_error "Please install Docker first. Visit: https://docs.docker.com/get-docker/" + exit 1 + fi + else + DOCKER_VERSION=$(docker --version | cut -d ' ' -f3 | tr -d ',') + log_success "Docker ${DOCKER_VERSION} is installed" + fi +} + +# Check if Docker Compose is installed +check_docker_compose() { + log_info "Checking Docker Compose installation..." + + detect_docker_cmd + + if ! ${DOCKER_CMD} compose version &> /dev/null; then + log_error "Docker Compose v2+ is not available." + if [ "$SILENT_MODE" = false ]; then + read -p "Would you like installation instructions? [y/N] " -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]]; then + echo "Please install Docker Compose v2+: https://docs.docker.com/compose/install/" + fi + fi + exit 1 + else + COMPOSE_VERSION=$(${DOCKER_CMD} compose version --short) + log_success "Docker Compose ${COMPOSE_VERSION} is installed" + fi +} + +# Detect whether docker needs sudo +detect_docker_cmd() { + DOCKER_CMD="docker" + + if ! docker ps >/dev/null 2>&1; then + # Try passwordless sudo first + if sudo -n docker ps >/dev/null 2>&1; then + DOCKER_CMD="sudo docker" + else + # Check if silent mode is enabled before prompting for sudo + if [ "$SILENT_MODE" = true ]; then + log_error "Docker requires sudo, but passwordless sudo is not available." + log_error "In --silent mode, you must have passwordless sudo permission or be in the docker group." + exit 1 + fi + + # Fall back to interactive sudo (may prompt for password) + if sudo docker ps >/dev/null 2>&1; then + DOCKER_CMD="sudo docker" + else + log_error "Docker is not accessible. Ensure you have permission to run Docker commands (with or without sudo)." + exit 1 + fi + fi + fi +} + +# Install Docker (support for multiple distros) +install_docker() { + log_info "Installing Docker..." + + if [ -f /etc/os-release ]; then + . /etc/os-release + case $ID in + ubuntu|debian) + # Use official Docker repository instead of convenience script for security + sudo apt-get update + sudo apt-get install -y ca-certificates curl gnupg + sudo install -m 0755 -d /etc/apt/keyrings + curl -fsSL "https://download.docker.com/linux/${ID}/gpg" | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg + sudo chmod a+r /etc/apt/keyrings/docker.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/${ID} ${VERSION_CODENAME} stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt-get update + sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + sudo systemctl start docker + sudo systemctl enable docker + sudo usermod -aG docker "$USER" + log_success "Docker installed. Please log out and back in for group changes to take effect." + ;; + fedora) + sudo dnf -y install dnf-plugins-core + sudo dnf config-manager --add-repo https://download.docker.com/linux/fedora/docker-ce.repo + sudo dnf install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + sudo systemctl start docker + sudo systemctl enable docker + sudo usermod -aG docker "$USER" + log_success "Docker installed. Please log out and back in for group changes to take effect." + ;; + centos|rhel|almalinux|rocky) + sudo yum install -y yum-utils + sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo + sudo yum install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + sudo systemctl start docker + sudo systemctl enable docker + sudo usermod -aG docker "$USER" + log_success "Docker installed. Please log out and back in for group changes to take effect." + ;; + opensuse*|sles) + sudo zypper install -y docker docker-compose + sudo systemctl start docker + sudo systemctl enable docker + sudo usermod -aG docker "$USER" + log_success "Docker installed. Please log out and back in for group changes to take effect." + ;; + *) + log_error "Automatic installation not supported for $ID" + log_info "Please install Docker manually:" + log_info " Visit: https://docs.docker.com/get-docker/" + exit 1 + ;; + esac + else + log_error "Cannot detect OS. Please install Docker manually." + log_info "Visit: https://docs.docker.com/get-docker/" + exit 1 + fi +} + +# Check system requirements +check_requirements() { + log_info "Checking system requirements..." + + # Check available memory + if command -v free &> /dev/null; then + TOTAL_MEM=$(free -g | awk '/^Mem:/{print $2}') + if [ "$ENABLE_ELASTIC" = true ] && [ "$TOTAL_MEM" -lt 16 ]; then + log_warning "Elasticsearch requires at least 16GB RAM. You have ${TOTAL_MEM}GB." + if [ "$SILENT_MODE" = false ]; then + read -p "Continue anyway? [y/N] " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + exit 1 + fi + fi + fi + fi + + check_docker + check_docker_compose + + # Check if git is installed + if ! command -v git &> /dev/null; then + log_warning "Git is not installed." + log_info "Some features (like update) will not work." + fi + + log_success "All requirements met" +} + +# Initialize environment files +init_env_files() { + log_info "Initializing environment files..." + + # Handle env_file + if [ -f "${SCRIPT_DIR}/docker/env_file" ]; then + if [ "$SILENT_MODE" = true ]; then + # Always create backup even in silent mode to protect user configs + local backup_name="${SCRIPT_DIR}/docker/env_file.backup.$(date +%Y%m%d_%H%M%S)" + cp "${SCRIPT_DIR}/docker/env_file" "$backup_name" + log_info "Backed up existing docker/env_file to $backup_name" + cp "${SCRIPT_DIR}/docker/env_file_template" "${SCRIPT_DIR}/docker/env_file" + log_success "Regenerated docker/env_file from template" + else + log_warning "docker/env_file already exists" + read -p "Do you want to regenerate it? This will backup the existing file. [y/N] " -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]]; then + mv "${SCRIPT_DIR}/docker/env_file" "${SCRIPT_DIR}/docker/env_file.backup.$(date +%Y%m%d_%H%M%S)" + cp "${SCRIPT_DIR}/docker/env_file_template" "${SCRIPT_DIR}/docker/env_file" + log_success "Backed up old file and created new docker/env_file from template" + else + log_info "Keeping existing docker/env_file" + fi + fi + else + if [ -f "${SCRIPT_DIR}/docker/env_file_template" ]; then + cp "${SCRIPT_DIR}/docker/env_file_template" "${SCRIPT_DIR}/docker/env_file" + log_success "Created docker/env_file from template" + + if [ "$SILENT_MODE" = false ] && [ -z "$ELASTIC_ENDPOINT" ]; then + log_warning "Please edit docker/env_file and configure required settings" + fi + else + log_error "docker/env_file_template not found" + exit 1 + fi + fi + + # Set Elasticsearch endpoint if provided + if [ -n "$ELASTIC_ENDPOINT" ]; then + # Escape ampersand to avoid sed interpreting it as the matched string + local escaped_endpoint="${ELASTIC_ENDPOINT//&/\\&}" + if grep -q "^ELASTIC_ENDPOINT=" "${SCRIPT_DIR}/docker/env_file"; then + sed -i "s|^ELASTIC_ENDPOINT=.*|ELASTIC_ENDPOINT=${escaped_endpoint}|" "${SCRIPT_DIR}/docker/env_file" + else + echo "ELASTIC_ENDPOINT=${ELASTIC_ENDPOINT}" >> "${SCRIPT_DIR}/docker/env_file" + fi + log_success "Configured ELASTIC_ENDPOINT in docker/env_file" + fi + + # Handle env_file_postgres + if [ -f "${SCRIPT_DIR}/docker/env_file_postgres" ]; then + if [ "$SILENT_MODE" = true ]; then + # Always create backup even in silent mode to protect user configs + local backup_name="${SCRIPT_DIR}/docker/env_file_postgres.backup.$(date +%Y%m%d_%H%M%S)" + cp "${SCRIPT_DIR}/docker/env_file_postgres" "$backup_name" + log_info "Backed up existing docker/env_file_postgres to $backup_name" + cp "${SCRIPT_DIR}/docker/env_file_postgres_template" "${SCRIPT_DIR}/docker/env_file_postgres" + log_success "Regenerated docker/env_file_postgres from template" + else + log_warning "docker/env_file_postgres already exists" + read -p "Do you want to regenerate it? This will backup the existing file. [y/N] " -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]]; then + mv "${SCRIPT_DIR}/docker/env_file_postgres" "${SCRIPT_DIR}/docker/env_file_postgres.backup.$(date +%Y%m%d_%H%M%S)" + cp "${SCRIPT_DIR}/docker/env_file_postgres_template" "${SCRIPT_DIR}/docker/env_file_postgres" + log_success "Backed up old file and created new docker/env_file_postgres from template" + else + log_info "Keeping existing docker/env_file_postgres" + fi + fi + else + if [ -f "${SCRIPT_DIR}/docker/env_file_postgres_template" ]; then + cp "${SCRIPT_DIR}/docker/env_file_postgres_template" "${SCRIPT_DIR}/docker/env_file_postgres" + log_success "Created docker/env_file_postgres from template" + else + log_error "docker/env_file_postgres_template not found" + exit 1 + fi + fi + + log_success "Environment files initialized" + + # Save .env configuration (COMPOSE_FILE, COMPOSE_PROJECT_NAME, etc.) + if [ -f "${SCRIPT_DIR}/.env" ]; then + if [ "$SILENT_MODE" = false ]; then + log_warning ".env already exists" + read -p "Do you want to regenerate it? This will backup the existing file. [y/N] " -n 1 -r + echo + if [[ $REPLY =~ ^[Yy]$ ]]; then + mv "${SCRIPT_DIR}/.env" "${SCRIPT_DIR}/.env.backup.$(date +%Y%m%d_%H%M%S)" + save_config + else + log_info "Keeping existing .env" + fi + else + local backup_name="${SCRIPT_DIR}/.env.backup.$(date +%Y%m%d_%H%M%S)" + cp "${SCRIPT_DIR}/.env" "$backup_name" + log_info "Backed up existing .env to $backup_name" + save_config + fi + else + save_config + fi +} + +# Execute docker compose command +# --project-directory points to docker/ so that relative env_file paths +# in compose files resolve correctly. --env-file loads .env from the +# repo root for COMPOSE_FILE, COMPOSE_PROJECT_NAME, etc. +execute_compose() { + local cmd=$1 + shift + + log_info "Executing: docker compose $cmd $*" + + detect_docker_cmd + ${DOCKER_CMD} compose --project-directory "${SCRIPT_DIR}/docker" \ + --env-file "${SCRIPT_DIR}/.env" "$cmd" "$@" +} + +# Checkout the appropriate git branch/tag for the deployment mode +checkout_git_ref() { + if [ ! -d "${SCRIPT_DIR}/.git" ]; then + return + fi + if ! command -v git &> /dev/null; then + return + fi + + local target_ref="main" + if [ "$ENV_MODE" = "dev" ]; then + target_ref="develop" + elif [ "$USE_VERSION" = true ]; then + if [ "$GREEDYBEAR_VERSION" = "stag" ]; then + target_ref="develop" + else + target_ref="$GREEDYBEAR_VERSION" + fi + fi + + local current_ref + current_ref=$(git -C "$SCRIPT_DIR" branch --show-current 2>/dev/null || git -C "$SCRIPT_DIR" rev-parse --short HEAD) + + if [ "$current_ref" = "$target_ref" ]; then + log_info "Already on '$target_ref'" + return + fi + + if [ -n "$(git -C "$SCRIPT_DIR" status --porcelain)" ]; then + if [ "$SILENT_MODE" = true ]; then + log_warning "Uncommitted changes detected. Force checking out '$target_ref'..." + else + log_warning "You have uncommitted changes in the repository." + read -r -p "Continue checking out '$target_ref'? Uncommitted changes may be lost. [y/N] " response + if [[ ! "$response" =~ ^[Yy]$ ]]; then + log_info "Checkout aborted." + exit 0 + fi + fi + fi + + log_info "Checking out '$target_ref'..." + if git -C "$SCRIPT_DIR" checkout "$target_ref"; then + log_success "Checked out '$target_ref'" + else + log_error "Failed to checkout '$target_ref'. Please check your git state." + exit 1 + fi +} + +# Initialize GreedyBear +cmd_init() { + log_info "Initializing GreedyBear..." + + check_requirements + checkout_git_ref + init_env_files + + log_success "Initialization complete!" + echo + log_info "Next steps:" + if [ "$SILENT_MODE" = false ]; then + log_info " 1. Review and configure docker/env_file with your settings" + log_info " 2. Start GreedyBear with: ./gbctl up" + else + log_info " Start GreedyBear with: ./gbctl up" + fi +} + +# Check for version downgrade +check_downgrade() { + # Only check if a specific version is requested + if [ "$USE_VERSION" != true ]; then + return + fi + + detect_docker_cmd + local docker_cmd="$DOCKER_CMD" + + # Check if uwsgi container exists + local container_name="${PROJECT_NAME}_uwsgi" + if ! ${docker_cmd} ps -a --format '{{.Names}}' | grep -q "^${container_name}$"; then + return + fi + + # Get current image + local current_image + current_image=$(${docker_cmd} inspect --format='{{.Config.Image}}' "$container_name" 2>/dev/null || echo "") + + if [ -z "$current_image" ]; then + return + fi + + # Extract tag (part after :) + local current_tag="${current_image##*:}" + + # If current tag is "prod", "latest" or matches requested, skip + # We can't reliably compare "prod"/"latest" with semantic versions + if [ "$current_tag" = "prod" ] || [ "$current_tag" = "latest" ] || [ "$current_tag" = "$GREEDYBEAR_VERSION" ]; then + return + fi + + # Check for downgrade using sort -V + local lowest_version + lowest_version=$(echo -e "${current_tag}\n${GREEDYBEAR_VERSION}" | sort -V | head -n1) + + if [ "$lowest_version" = "$GREEDYBEAR_VERSION" ] && [ "$current_tag" != "$GREEDYBEAR_VERSION" ]; then + log_warning "POTENTIAL DOWNGRADE DETECTED!" + log_warning "Current running version: $current_tag" + log_warning "Target version: $GREEDYBEAR_VERSION" + log_warning "Downgrading may cause database incompatibility issues since the database" + log_warning "might have been migrated to the newer version." + + if [ "$SILENT_MODE" = false ]; then + read -p "Are you sure you want to proceed? [y/N] " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + log_info "Operation cancelled" + exit 1 + fi + else + log_error "Aborting downgrade in silent mode to prevent data corruption." + exit 1 + fi + fi +} + +# Start services +cmd_up() { + log_info "Starting GreedyBear services..." + + check_downgrade + check_git_version_mismatch + + execute_compose up -d "$@" + log_success "GreedyBear services started successfully!" +} + +# Check for git version mismatch +check_git_version_mismatch() { + # Only check if we are in a git repo and not pinning a specific version + if [ -d "${SCRIPT_DIR}/.git" ] && [ "$USE_VERSION" != true ]; then + if command -v git &> /dev/null; then + local current_branch + current_branch=$(git -C "$SCRIPT_DIR" branch --show-current) + + # If we are on main/master but running dev, or vice versa + # This is a heuristic check + if [ "$ENV_MODE" = "prod" ] && [ "$current_branch" != "main" ] && [ "$current_branch" != "master" ]; then + log_warning "Version Mismatch Warning:" + log_warning " You are running in PROD mode but your git branch is '$current_branch'." + log_warning " Production usually runs on 'main' or 'master'." + elif [ "$ENV_MODE" = "dev" ] && ([ "$current_branch" = "main" ] || [ "$current_branch" = "master" ]); then + log_info "Note: You are running in DEV mode on the '$current_branch' branch." + fi + fi + fi +} + + +# Stop and remove services +cmd_down() { + log_info "Stopping GreedyBear services..." + execute_compose down "$@" + log_success "GreedyBear services stopped" +} + +# Stop services +cmd_stop() { + log_info "Stopping GreedyBear services..." + execute_compose stop "$@" + log_success "GreedyBear services stopped" +} + +# Restart services +cmd_restart() { + log_info "Restarting GreedyBear services..." + execute_compose restart "$@" + log_success "GreedyBear services restarted" +} + +# View logs +cmd_logs() { + if [ $# -gt 0 ] && [ "$1" == "app" ]; then + shift + log_info "Tailing Django application logs (Ctrl+C to exit)..." + + detect_docker_cmd + local container_name="${PROJECT_NAME}_uwsgi" + + # Check if container is running + if ! ${DOCKER_CMD} ps --format '{{.Names}}' | grep -q "^${container_name}$"; then + log_error "Container $container_name is not running." + exit 1 + fi + + # Tail the log file inside the container + ${DOCKER_CMD} exec -it "$container_name" tail -f /var/log/greedybear/django/greedybear.log + else + execute_compose logs "$@" + fi +} + +# List services +cmd_ps() { + execute_compose ps "$@" +} + +# Update GreedyBear +cmd_update() { + log_info "Updating GreedyBear..." + + if [ -d "${SCRIPT_DIR}/.git" ]; then + if [ "$USE_VERSION" = true ] && [ "$GREEDYBEAR_VERSION" != "stag" ]; then + # Pinned to a version tag — don't pull, just check for newer releases + git -C "$SCRIPT_DIR" fetch --tags --quiet 2>/dev/null + local latest_tag + latest_tag=$(git -C "$SCRIPT_DIR" tag | sed 's/^v//' | sort -t. -k1,1n -k2,2n -k3,3n | tail -1) + if [ -n "$latest_tag" ] && [ "$latest_tag" != "$GREEDYBEAR_VERSION" ]; then + log_warning "Pinned to ${GREEDYBEAR_VERSION}, but ${latest_tag} is available" + log_info "To upgrade, run: ./gbctl init --release ${latest_tag}" + fi + else + # Tracking a branch (develop, main, or stag) — pull latest + log_info "Pulling latest code from git..." + if git -C "$SCRIPT_DIR" pull; then + log_success "Code updated successfully" + else + log_error "Failed to pull latest code. Please check your git configuration." + exit 1 + fi + fi + else + log_warning "Not a git repository. Skipping code update." + fi + + # Pull latest images + log_info "Pulling latest Docker images..." + execute_compose pull + + # Restart services using cmd_up to ensure version env vars are applied + log_info "Restarting services with new images..." + cmd_up "$@" + + log_success "GreedyBear updated successfully!" +} + +# Build images +cmd_build() { + log_info "Building GreedyBear images..." + execute_compose build "$@" + log_success "Build complete" +} + +# Pull images +cmd_pull() { + log_info "Pulling GreedyBear images..." + execute_compose pull "$@" + log_success "Pull complete" +} + +# Backup PostgreSQL database +cmd_backup() { + log_info "Creating backup..." + + local backup_dir="${SCRIPT_DIR}/backups" + local timestamp=$(date +%Y%m%d_%H%M%S) + local backup_file="${backup_dir}/greedybear_backup_${timestamp}" + + # Securely create backup directory + if [ ! -d "$backup_dir" ]; then + mkdir -p "$backup_dir" + chmod 700 "$backup_dir" + fi + + # Source PostgreSQL credentials from env file + local pg_user="user" + local pg_db="greedybear_db" + if [ -f "${SCRIPT_DIR}/docker/env_file_postgres" ]; then + pg_user=$(grep -E '^POSTGRES_USER=' "${SCRIPT_DIR}/docker/env_file_postgres" | cut -d'=' -f2 || echo "user") + pg_db=$(grep -E '^POSTGRES_DB=' "${SCRIPT_DIR}/docker/env_file_postgres" | cut -d'=' -f2 || echo "greedybear_db") + fi + + detect_docker_cmd + local docker_cmd="$DOCKER_CMD" + + # Check if postgres container is running + if ! ${docker_cmd} ps --format '{{.Names}}' | grep -q "${PROJECT_NAME}_postgres"; then + log_error "PostgreSQL container is not running. Start services first." + exit 1 + fi + + # Backup PostgreSQL database + # Write dump to a file inside the container to avoid stdout corruption, + # then copy it out to the host. + log_info "Backing up PostgreSQL database..." + set +e # Temporarily disable exit on error to handle failures gracefully + ${docker_cmd} exec ${PROJECT_NAME}_postgres pg_dump -U "$pg_user" -f /tmp/gb_backup.sql "$pg_db" + backup_status=$? + set -e # Re-enable exit on error + + if [ $backup_status -eq 0 ]; then + ${docker_cmd} cp ${PROJECT_NAME}_postgres:/tmp/gb_backup.sql "${backup_file}.sql" + ${docker_cmd} exec ${PROJECT_NAME}_postgres rm -f /tmp/gb_backup.sql + gzip "${backup_file}.sql" + chmod 600 "${backup_file}.sql.gz" + log_success "Database backup created: ${backup_file}.sql.gz" + else + ${docker_cmd} exec ${PROJECT_NAME}_postgres rm -f /tmp/gb_backup.sql 2>/dev/null || true + log_error "Database backup failed" + exit 1 + fi + + # Backup volumes info + log_info "Saving volumes information..." + ${docker_cmd} volume ls --filter name=${PROJECT_NAME} > "${backup_file}_volumes.txt" + + log_success "Backup complete: ${backup_file}.sql.gz" + log_info "To restore: ./gbctl restore ${backup_file}.sql.gz" +} + +# Restore from backup +cmd_restore() { + if [ $# -eq 0 ]; then + log_error "Please specify backup file to restore" + log_info "Usage: ./gbctl restore backups/greedybear_backup_YYYYMMDD_HHMMSS.sql.gz" + log_info "Available backups in ${SCRIPT_DIR}/backups:" + ls -lh "${SCRIPT_DIR}/backups/"*.sql.gz 2>/dev/null || log_warning "No backups found" + exit 1 + fi + + local backup_file=$1 + + if [ ! -f "$backup_file" ]; then + log_error "Backup file not found: $backup_file" + exit 1 + fi + + detect_docker_cmd + local docker_cmd="$DOCKER_CMD" + + # Source PostgreSQL credentials from env file + local pg_user="user" + local pg_db="greedybear_db" + if [ -f "${SCRIPT_DIR}/docker/env_file_postgres" ]; then + pg_user=$(grep -E '^POSTGRES_USER=' "${SCRIPT_DIR}/docker/env_file_postgres" | cut -d'=' -f2 || echo "user") + pg_db=$(grep -E '^POSTGRES_DB=' "${SCRIPT_DIR}/docker/env_file_postgres" | cut -d'=' -f2 || echo "greedybear_db") + fi + + # Check if postgres container is running + if ! ${docker_cmd} ps --format '{{.Names}}' | grep -q "${PROJECT_NAME}_postgres"; then + log_error "PostgreSQL container is not running. Start services first." + exit 1 + fi + + if [ "$SILENT_MODE" = false ]; then + log_warning "This will overwrite the current database!" + read -p "Are you sure you want to restore from backup? [y/N] " -n 1 -r + echo + if [[ ! $REPLY =~ ^[Yy]$ ]]; then + log_info "Restore cancelled" + exit 0 + fi + fi + + log_info "Restoring database from: $backup_file" + + # Drop and recreate the database to avoid duplicate object errors + log_info "Dropping and recreating database..." + ${docker_cmd} exec ${PROJECT_NAME}_postgres dropdb -U "$pg_user" "$pg_db" + ${docker_cmd} exec ${PROJECT_NAME}_postgres createdb -U "$pg_user" "$pg_db" + + # Decompress and restore + set +e # Temporarily disable exit on error to handle failures gracefully + if [[ $backup_file == *.gz ]]; then + # Use explicit pipefail check for gunzip | docker exec pipeline + set -o pipefail + gunzip -c "$backup_file" | ${docker_cmd} exec -i ${PROJECT_NAME}_postgres psql -U "$pg_user" "$pg_db" + restore_status=$? + set -o pipefail # Keep pipefail enabled (script default) + else + ${docker_cmd} exec -i ${PROJECT_NAME}_postgres psql -U "$pg_user" "$pg_db" < "$backup_file" + restore_status=$? + fi + set -e # Re-enable exit on error + + if [ $restore_status -eq 0 ]; then + log_success "Database restored successfully" + else + log_error "Restore failed" + exit 1 + fi +} + +# Health check all services +cmd_health() { + log_info "Checking GreedyBear services health..." + echo + + detect_docker_cmd + local docker_cmd="$DOCKER_CMD" + + # Check each service + local services=("postgres" "uwsgi" "nginx" "rabbitmq" "celery_beat" "celery_worker_default") + local all_healthy=true + + for service in "${services[@]}"; do + local container_name="${PROJECT_NAME}_${service}" + + if ${docker_cmd} ps --format '{{.Names}}' | grep -q "^${container_name}$"; then + local status=$(${docker_cmd} inspect --format='{{.State.Status}}' "$container_name" 2>/dev/null) + local health=$(${docker_cmd} inspect --format='{{if .State.Health}}{{.State.Health.Status}}{{else}}no healthcheck{{end}}' "$container_name" 2>/dev/null) + + if [ "$status" = "running" ]; then + if [ "$health" = "healthy" ] || [ "$health" = "no healthcheck" ]; then + echo -e "${GREEN}✓${NC} $service: running" + else + echo -e "${YELLOW}!${NC} $service: running (health: $health)" + all_healthy=false + fi + else + echo -e "${RED}✗${NC} $service: $status" + all_healthy=false + fi + else + echo -e "${RED}✗${NC} $service: not running" + all_healthy=false + fi + done + + echo + if [ "$all_healthy" = true ]; then + log_success "All services are healthy" + return 0 + else + log_warning "Some services have issues" + log_info "Run './gbctl logs ' for details" + return 1 + fi +} + +# Clean all data and reset +cmd_clean() { + log_warning "This will remove ALL GreedyBear data, containers, and volumes!" + log_warning "This action is IRREVERSIBLE!" + + # Always require explicit --force flag for safety + if [ "$FORCE_CLEAN" = false ]; then + log_error "The clean command requires the --force flag to prevent accidental data loss" + log_info "Usage: ./gbctl clean --force" + exit 1 + fi + + if [ "$SILENT_MODE" = false ]; then + read -p "Type 'yes' to confirm complete removal: " confirmation + if [ "$confirmation" != "yes" ]; then + log_info "Clean cancelled" + exit 0 + fi + fi + + log_info "Stopping and removing all containers..." + execute_compose down -v + + log_info "Removing environment and configuration files..." + rm -f "${SCRIPT_DIR}/docker/env_file" "${SCRIPT_DIR}/docker/env_file_postgres" "${SCRIPT_DIR}/.env" + + log_success "GreedyBear has been completely removed" + log_info "To reinstall, run: ./gbctl init" +} + +# Create admin user +cmd_create_admin() { + log_info "Creating Django superuser..." + + detect_docker_cmd + local docker_cmd="$DOCKER_CMD" + local container_name="${PROJECT_NAME}_uwsgi" + + # Check if uwsgi container is running + if ! ${docker_cmd} ps --format '{{.Names}}' | grep -q "^${container_name}$"; then + log_error "GreedyBear uWSGI container is not running. Please start services first." + exit 1 + fi + + # Check if we have arguments for silent mode (from flags or environment variables) + if [ -n "$ADMIN_USERNAME" ] && [ -n "$ADMIN_PASSWORD" ] && [ -n "$ADMIN_EMAIL" ]; then + log_info "Creating superuser in non-interactive mode..." + + if ${docker_cmd} exec -e DJANGO_SUPERUSER_PASSWORD="$ADMIN_PASSWORD" \ + -e DJANGO_SUPERUSER_USERNAME="$ADMIN_USERNAME" \ + -e DJANGO_SUPERUSER_EMAIL="$ADMIN_EMAIL" \ + -e DJANGO_SUPERUSER_FIRST_NAME="admin" \ + -e DJANGO_SUPERUSER_LAST_NAME="user" \ + "$container_name" python3 manage.py createsuperuser --noinput; then + log_success "Superuser '$ADMIN_USERNAME' created successfully!" + else + log_error "Failed to create superuser." + exit 1 + fi + else + # Fallback to interactive mode if ANY argument is missing + if [ "$SILENT_MODE" = true ] && ( [ -z "$ADMIN_USERNAME" ] || [ -z "$ADMIN_PASSWORD" ] || [ -z "$ADMIN_EMAIL" ] ); then + log_error "Silent mode requires username, password, and email." + log_info "Provide them via flags (--username, --password, --email)" + log_info "OR via environment variables (GB_ADMIN_USERNAME, GB_ADMIN_PASSWORD, GB_ADMIN_EMAIL)" + exit 1 + fi + + log_info "Entering interactive mode..." + log_info "Please follow the prompts to create your superuser." + + # Use -it for interactive terminal + ${docker_cmd} exec -it "$container_name" python3 manage.py createsuperuser + fi +} + +# Parse arguments +parse_args() { + # Set defaults + ENV_MODE="prod" + ENABLE_HTTPS=false + ENABLE_ELASTIC=false + USE_VERSION=false + FORCE_CLEAN=false + COMMAND="" + PASSED_FLAGS=() + + # Check for help (early check before loading config) + if [[ $# -eq 0 ]] || [[ "$1" == "-h" ]] || [[ "$1" == "--help" ]]; then + print_help + exit 0 + fi + + # First argument should be command + if [[ -v cmd_arguments["$1"] ]] && [[ ${cmd_arguments["$1"]} ]]; then + COMMAND=$1 + shift + else + log_error "Invalid command: $1" + print_help + exit 1 + fi + + # Load configuration (unless we are initializing) + # We load config AFTER setting command but BEFORE parsing other flags + # This allows flags to override config + if [[ "$COMMAND" != "init" ]]; then + load_config + fi + + # Parse remaining arguments + while [[ $# -gt 0 ]]; do + case $1 in + --dev) + ENV_MODE="dev" + PASSED_FLAGS+=("--dev") + shift + ;; + --https) + ENABLE_HTTPS=true + PASSED_FLAGS+=("--https") + shift + ;; + --elastic-local) + ENABLE_ELASTIC=true + PASSED_FLAGS+=("--elastic-local") + shift + ;; + --release) + if [[ -z "${2:-}" || "$2" == -* ]]; then + echo -e "${RED}Error:${NC} --release requires a non-empty value" >&2 + exit 1 + fi + GREEDYBEAR_VERSION=$2 + USE_VERSION=true # Implicitly enable version override when version is specified + PASSED_FLAGS+=("--release") + shift 2 + ;; + --elastic-endpoint) + if [[ -z "${2:-}" || "$2" == -* ]]; then + echo -e "${RED}Error:${NC} --elastic-endpoint requires a non-empty value" >&2 + exit 1 + fi + ELASTIC_ENDPOINT=$2 + PASSED_FLAGS+=("--elastic-endpoint") + shift 2 + ;; + -s|--silent) + SILENT_MODE=true + PASSED_FLAGS+=("--silent") + shift + ;; + --force) + FORCE_CLEAN=true + PASSED_FLAGS+=("--force") + shift + ;; + --username) + if [[ -z "${2:-}" ]]; then + echo -e "${RED}Error:${NC} --username requires a non-empty value" >&2 + exit 1 + fi + ADMIN_USERNAME=$2 + PASSED_FLAGS+=("--username") + shift 2 + ;; + --password) + if [[ -z "${2:-}" ]]; then + echo -e "${RED}Error:${NC} --password requires a non-empty value" >&2 + exit 1 + fi + ADMIN_PASSWORD=$2 + PASSED_FLAGS+=("--password") + shift 2 + ;; + --email) + if [[ -z "${2:-}" ]]; then + echo -e "${RED}Error:${NC} --email requires a non-empty value" >&2 + exit 1 + fi + ADMIN_EMAIL=$2 + PASSED_FLAGS+=("--email") + shift 2 + ;; + -h|--help) + print_help + exit 0 + ;; + *) + # Pass unknown args to docker compose + break + ;; + esac + done + + # Store remaining args for docker compose + EXTRA_ARGS=("$@") +} + +# Validate that only allowed flags were passed for the given command +validate_flags() { + local cmd="$1" + shift + local allowed=("$@") + + for flag in "${PASSED_FLAGS[@]}"; do + local valid=false + for a in "${allowed[@]}"; do + if [ "$flag" = "$a" ]; then + valid=true + break + fi + done + if [ "$valid" = false ]; then + log_error "Flag '$flag' is not valid for '$cmd' command" + log_info "Run './gbctl --help' for usage information" + exit 1 + fi + done +} + +# Main execution +main() { + parse_args "$@" + + # Validate flags and execute command + local init_flags=(--dev --https --elastic-local --release --elastic-endpoint --silent) + + # Reject conflicting flags + if [ "$ENV_MODE" = "dev" ] && [ "$USE_VERSION" = true ]; then + log_error "--dev and --release are mutually exclusive" + log_info "--dev builds images locally, --release pins a remote image tag" + exit 1 + fi + if [ "$ENABLE_ELASTIC" = true ] && [ -n "$ELASTIC_ENDPOINT" ]; then + log_error "--elastic-local and --elastic-endpoint are mutually exclusive" + log_info "--elastic-local runs a local container, --elastic-endpoint connects to an external instance" + exit 1 + fi + + case $COMMAND in + init) + validate_flags init "${init_flags[@]}" + cmd_init "${EXTRA_ARGS[@]}" + ;; + up|start) + validate_flags "$COMMAND" --silent + cmd_up "${EXTRA_ARGS[@]}" + ;; + down) + validate_flags down --silent + cmd_down "${EXTRA_ARGS[@]}" + ;; + stop) + validate_flags stop --silent + cmd_stop "${EXTRA_ARGS[@]}" + ;; + restart) + validate_flags restart --silent + cmd_restart "${EXTRA_ARGS[@]}" + ;; + logs) + validate_flags logs --silent + cmd_logs "${EXTRA_ARGS[@]}" + ;; + ps) + validate_flags ps --silent + cmd_ps "${EXTRA_ARGS[@]}" + ;; + update) + validate_flags update --silent + cmd_update "${EXTRA_ARGS[@]}" + ;; + build) + validate_flags build --silent + cmd_build "${EXTRA_ARGS[@]}" + ;; + pull) + validate_flags pull --silent + cmd_pull "${EXTRA_ARGS[@]}" + ;; + backup) + validate_flags backup --silent + cmd_backup "${EXTRA_ARGS[@]}" + ;; + restore) + validate_flags restore --silent + cmd_restore "${EXTRA_ARGS[@]}" + ;; + health) + validate_flags health --silent + cmd_health "${EXTRA_ARGS[@]}" + ;; + clean) + validate_flags clean --force --silent + cmd_clean "${EXTRA_ARGS[@]}" + ;; + create-admin) + validate_flags create-admin --username --password --email --silent + cmd_create_admin "${EXTRA_ARGS[@]}" + ;; + *) + log_error "Unknown command: $COMMAND" + print_help + exit 1 + ;; + esac +} + +# Run main function +main "$@" diff --git a/greedybear/admin.py b/greedybear/admin.py index b3752bf1..84c27260 100644 --- a/greedybear/admin.py +++ b/greedybear/admin.py @@ -31,7 +31,9 @@ class TorExitNodeModelAdmin(admin.ModelAdmin): @admin.register(Sensor) class SensorsModelAdmin(admin.ModelAdmin): - list_display = [field.name for field in Sensor._meta.get_fields()] + list_display = ["id", "address"] + search_fields = ["address"] + search_help_text = ["search for the sensor IP address"] @admin.register(Statistics) @@ -122,6 +124,7 @@ class IOCModelAdmin(admin.ModelAdmin): "scanner", "payload_request", "general_honeypots", + "sensor_list", "ip_reputation", "firehol_categories", "asn", @@ -138,12 +141,19 @@ class IOCModelAdmin(admin.ModelAdmin): search_fields = ["name", "related_ioc__name"] search_help_text = ["search for the IP address source"] raw_id_fields = ["related_ioc"] - filter_horizontal = ["general_honeypot"] + filter_horizontal = ["general_honeypot", "sensors"] inlines = [SessionInline] def general_honeypots(self, ioc): return ", ".join([str(element) for element in ioc.general_honeypot.all()]) + def sensor_list(self, ioc): + return ", ".join([str(sensor.address) for sensor in ioc.sensors.all()]) + + def get_queryset(self, request): + """Override to prefetch related sensors and honeypots, avoiding N+1 queries.""" + return super().get_queryset(request).prefetch_related("sensors", "general_honeypot") + @admin.register(GeneralHoneypot) class GeneralHoneypotAdmin(admin.ModelAdmin): diff --git a/greedybear/celery.py b/greedybear/celery.py index 53b3c383..8a10d099 100644 --- a/greedybear/celery.py +++ b/greedybear/celery.py @@ -58,30 +58,25 @@ def setup_loggers(*args, **kwargs): hp_extraction_interval = EXTRACTION_INTERVAL app.conf.beat_schedule = { - # every 10 minutes or according to EXTRACTION_INTERVAL + # =========================================== + # TIMING-CRITICAL: Extraction Task + # Runs every 10 minutes (or EXTRACTION_INTERVAL) + # Slots: :00, :10, :20, :30, :40, :50 + # =========================================== "extract_all": { "task": "greedybear.tasks.extract_all", "schedule": crontab(minute=f"*/{hp_extraction_interval}"), "options": {"queue": "default", "countdown": 10}, }, - # once an hour - "monitor_honeypots": { - "task": "greedybear.tasks.monitor_honeypots", - "schedule": crontab(minute=18), - "options": {"queue": "default"}, - }, - # once an hour - "monitor_logs": { - "task": "greedybear.tasks.monitor_logs", - "schedule": crontab(minute=33), - "options": {"queue": "default"}, - }, + # =========================================== + # TIMING-CRITICAL: Training # SCORING # Important: # The training task must be run with a small offset after midnight (00:00) # to ensure training data aligns with complete calendar days. # The small offset is to make sure that the midnight extraction task is completed before training. # This way models learn from complete rather than partial day patterns, which is crucial for their performance. + # =========================================== "train_and_update": { "task": "greedybear.tasks.chain_train_and_update", # Sometimes this could start when the midnight extraction is not ended yet. @@ -89,37 +84,56 @@ def setup_loggers(*args, **kwargs): "schedule": crontab(hour=0, minute=int(hp_extraction_interval / 3 * 2)), "options": {"queue": "default"}, }, - # COMMANDS + # =========================================== + # HOURLY: Monitoring Tasks + # Run at :07 (avoiding extraction slots) + # =========================================== + "monitor_honeypots": { + "task": "greedybear.tasks.monitor_honeypots", + "schedule": crontab(minute=7), + "options": {"queue": "default"}, + }, + "monitor_logs": { + "task": "greedybear.tasks.monitor_logs", + "schedule": crontab(minute=7), + "options": {"queue": "default"}, + }, + # =========================================== + # DAILY/WEEKLY: Maintenance Tasks + # All bundled at 1:07 AM + # Timing not critical - Celery queues sequentially + # Avoids extraction slots at 1:00 and 1:10 + # =========================================== # once a day "command_clustering": { "task": "greedybear.tasks.cluster_commands", - "schedule": crontab(hour=1, minute=3), + "schedule": crontab(hour=1, minute=7), "options": {"queue": "default"}, }, # once a day "clean_up": { "task": "greedybear.tasks.clean_up_db", - "schedule": crontab(hour=2, minute=3), + "schedule": crontab(hour=1, minute=7), "options": {"queue": "default"}, }, "get_mass_scanners": { "task": "greedybear.tasks.get_mass_scanners", - "schedule": crontab(hour=4, minute=3, day_of_week=0), + "schedule": crontab(hour=1, minute=7, day_of_week=0), "options": {"queue": "default"}, }, "get_whatsmyip": { "task": "greedybear.tasks.get_whatsmyip", - "schedule": crontab(hour=4, minute=3, day_of_week=6), + "schedule": crontab(hour=1, minute=7, day_of_week=0), "options": {"queue": "default"}, }, "extract_firehol_lists": { "task": "greedybear.tasks.extract_firehol_lists", - "schedule": crontab(hour=4, minute=15, day_of_week=0), + "schedule": crontab(hour=1, minute=7, day_of_week=0), "options": {"queue": "default"}, }, "get_tor_exit_nodes": { "task": "greedybear.tasks.get_tor_exit_nodes", - "schedule": crontab(hour=4, minute=30, day_of_week=0), + "schedule": crontab(hour=1, minute=7, day_of_week=0), "options": {"queue": "default"}, }, } diff --git a/greedybear/consts.py b/greedybear/consts.py index fb3af390..c5b684e6 100644 --- a/greedybear/consts.py +++ b/greedybear/consts.py @@ -29,3 +29,9 @@ "password", "t-pot_ip_ext", ] + + +# we used this const to implement news feature +RSS_FEED_URL = "https://intelowlproject.github.io/feed.xml" +CACHE_KEY_GREEDYBEAR_NEWS = "greedybear_news" +CACHE_TIMEOUT_SECONDS = 60 * 60 diff --git a/greedybear/cronjobs/extraction/ioc_processor.py b/greedybear/cronjobs/extraction/ioc_processor.py index 286030c2..864965ca 100644 --- a/greedybear/cronjobs/extraction/ioc_processor.py +++ b/greedybear/cronjobs/extraction/ioc_processor.py @@ -26,12 +26,17 @@ def __init__(self, ioc_repo: IocRepository, sensor_repo: SensorRepository): self.ioc_repo = ioc_repo self.sensor_repo = sensor_repo - def add_ioc(self, ioc: IOC, attack_type: str, general_honeypot_name: str = None) -> IOC | None: + def add_ioc( + self, + ioc: IOC, + attack_type: str, + general_honeypot_name: str = None, + ) -> IOC | None: """ Process an IOC record. Filters out sensor IPs and whats-my-ip domains, then creates a new IOC record or updates an existing one. Associates the IOC with a - general honeypot if specified. + general honeypot and/or sensor if specified. Args: ioc: IOC instance to process. @@ -43,7 +48,7 @@ def add_ioc(self, ioc: IOC, attack_type: str, general_honeypot_name: str = None) """ self.log.info(f"processing ioc {ioc} for attack_type {attack_type}") - if ioc.name in self.sensor_repo.sensors: + if ioc.name in self.sensor_repo.cache: self.log.debug(f"not saved {ioc} because it is a sensor") return None @@ -55,7 +60,12 @@ def add_ioc(self, ioc: IOC, attack_type: str, general_honeypot_name: str = None) if ioc_record is None: # Create self.log.debug(f"{ioc} was not seen before - creating a new record") ioc_record = self.ioc_repo.save(ioc) - else: # Update + # Add sensors to newly saved IOC from temporary attribute. + # (See greedybear/cronjobs/extraction/utils.py for why we use this) + if hasattr(ioc, "_sensors_to_add") and ioc._sensors_to_add: + for sensor in ioc._sensors_to_add: + ioc_record.sensors.add(sensor) + else: # Update - sensors handled inside _merge_iocs self.log.debug(f"{ioc} is already known - updating record") ioc_record = self._merge_iocs(ioc_record, ioc) @@ -72,7 +82,7 @@ def add_ioc(self, ioc: IOC, attack_type: str, general_honeypot_name: str = None) def _merge_iocs(self, existing: IOC, new: IOC) -> IOC: """ Merge a new IOC's data into an existing record. - Updates timestamps, increments counters, and combines list fields. + Updates timestamps, increments counters, combines list fields, and adds sensors. Args: existing: The existing IOC record from the database. @@ -89,6 +99,13 @@ def _merge_iocs(self, existing: IOC, new: IOC) -> IOC: existing.ip_reputation = new.ip_reputation existing.asn = new.asn existing.login_attempts += new.login_attempts + + # Add sensors from new IOC (existing is already saved, so ManyToMany works). + # We retrieve sensors from the temporary attribute of the input IOC object. + if hasattr(new, "_sensors_to_add") and new._sensors_to_add: + for sensor in new._sensors_to_add: + existing.sensors.add(sensor) + return existing def _update_days_seen(self, ioc: IOC) -> IOC: diff --git a/greedybear/cronjobs/extraction/pipeline.py b/greedybear/cronjobs/extraction/pipeline.py index 9874cb5e..969df89e 100644 --- a/greedybear/cronjobs/extraction/pipeline.py +++ b/greedybear/cronjobs/extraction/pipeline.py @@ -72,10 +72,12 @@ def execute(self) -> int: # skip hits with non-existing or empty types (=honeypots) if "type" not in hit or not hit["type"].strip(): continue - # extract sensor + # extract sensor and include in hit dict + hit_dict = hit.to_dict() if "t-pot_ip_ext" in hit: - self.sensor_repo.add_sensor(hit["t-pot_ip_ext"]) - hits_by_honeypot[hit["type"]].append(hit.to_dict()) + sensor = self.sensor_repo.get_or_create_sensor(hit["t-pot_ip_ext"]) + hit_dict["_sensor"] = sensor # Include sensor object for strategies + hits_by_honeypot[hit["type"]].append(hit_dict) # 3. Extract using strategies for honeypot, hits in sorted(hits_by_honeypot.items()): diff --git a/greedybear/cronjobs/extraction/strategies/cowrie.py b/greedybear/cronjobs/extraction/strategies/cowrie.py index 71738559..6eb9e9ab 100644 --- a/greedybear/cronjobs/extraction/strategies/cowrie.py +++ b/greedybear/cronjobs/extraction/strategies/cowrie.py @@ -147,6 +147,9 @@ def _extract_possible_payload_in_messages(self, hits: list[dict]) -> None: type=get_ioc_type(payload_hostname), related_urls=[payload_url], ) + sensor = hit.get("_sensor") + if sensor: + ioc._sensors_to_add = [sensor] self.ioc_processor.add_ioc(ioc, attack_type=PAYLOAD_REQUEST, general_honeypot_name="Cowrie") self._add_fks(scanner_ip, payload_hostname) self.payloads_in_message += 1 @@ -181,6 +184,9 @@ def _get_url_downloads(self, hits: list[dict]) -> None: type=get_ioc_type(hostname), related_urls=[download_url], ) + sensor = hit.get("_sensor") + if sensor: + ioc._sensors_to_add = [sensor] ioc_record = self.ioc_processor.add_ioc(ioc, attack_type=PAYLOAD_REQUEST, general_honeypot_name="Cowrie") if ioc_record: self.added_url_downloads += 1 diff --git a/greedybear/cronjobs/extraction/utils.py b/greedybear/cronjobs/extraction/utils.py index abd59a44..fa0d8492 100644 --- a/greedybear/cronjobs/extraction/utils.py +++ b/greedybear/cronjobs/extraction/utils.py @@ -89,7 +89,7 @@ def get_firehol_categories(ip: str, extracted_ip) -> list[str]: def iocs_from_hits(hits: list[dict]) -> list[IOC]: """ - Convert Elasticsearch hits into IOC objects. + Convert Elasticsearch hits into IOC objects with associated sensors. Groups hits by source IP, filters out non-global addresses, and constructs IOC objects with aggregated data. Enriches IOCs with FireHol categories at creation time to ensure @@ -113,6 +113,12 @@ def iocs_from_hits(hits: list[dict]) -> list[IOC]: firehol_categories = get_firehol_categories(ip, extracted_ip) + # Collect unique sensors from hits, deduplicated by sensor ID + sensors_map = {hit["_sensor"].id: hit["_sensor"] for hit in hits if hit.get("_sensor") is not None and getattr(hit["_sensor"], "id", None)} + sensors = list(sensors_map.values()) + # Sort sensors by ID for consistent processing order + sensors.sort(key=lambda s: s.id) + ioc = IOC( name=ip, type=get_ioc_type(ip), @@ -123,6 +129,11 @@ def iocs_from_hits(hits: list[dict]) -> list[IOC]: login_attempts=len(hits) if hits[0].get("type", "") == "Heralding" else 0, firehol_categories=firehol_categories, ) + # Attach sensors to temporary attribute for later processing. + # We cannot use `ioc.sensors.add()` here because the IOC instance is not yet saved + # to the database, and Django requires an ID for M2M relationships. + ioc._sensors_to_add = sensors + timestamps = [hit["@timestamp"] for hit in hits if "@timestamp" in hit] if timestamps: ioc.first_seen = datetime.fromisoformat(min(timestamps)) diff --git a/greedybear/cronjobs/repositories/sensor.py b/greedybear/cronjobs/repositories/sensor.py index 0e111789..12a35bbd 100644 --- a/greedybear/cronjobs/repositories/sensor.py +++ b/greedybear/cronjobs/repositories/sensor.py @@ -10,49 +10,39 @@ class SensorRepository: Repository for data access to the set of T-Pot sensors with in-memory caching. The cache is populated once from the database at initialization and updated - on successful additions. + on successful additions. Stores Sensor objects for efficient retrieval. """ def __init__(self): """Initialize the repository and populate the cache from the database.""" self.log = logging.getLogger(f"{__name__}.{self.__class__.__name__}") - self.cache = set() + self.cache: dict[str, Sensor] = {} self._fill_cache() - @property - def sensors(self) -> set: + def get_or_create_sensor(self, ip: str) -> Sensor | None: """ - Get the set of known sensor IP addresses. - - Returns: - Set of IP address strings for all known sensors. - """ - return self.cache - - def add_sensor(self, ip: str) -> bool: - """ - Add a new sensor IP address. - Validates that the IP is not already known and is a valid IP address - before writing it to the database and updating the cache. + Get an existing sensor or create a new one. + Validates that the IP is a valid IP address before writing it to the + database and updating the cache. Args: - ip: IP address string to add . + ip: IP address string. Returns: - True if the sensor was added, False if already known or invalid. + Sensor object if valid, None if invalid IP format. """ if ip in self.cache: - return False + return self.cache[ip] if get_ioc_type(ip) != IP: self.log.debug(f"{ip} is not an IP address - won't add as a sensor") - return False - sensor = Sensor(address=ip) - sensor.save() - self.cache.add(ip) - self.log.info(f"added sensor {ip} to the database") - return True + return None + sensor, created = Sensor.objects.get_or_create(address=ip) + self.cache[ip] = sensor + if created: + self.log.info(f"added sensor {ip} to the database") + return sensor def _fill_cache(self) -> None: - """Load sensor addresses from the database into the cache.""" + """Load sensor objects from the database into the cache.""" self.log.debug("populating sensor cache") - self.cache = {s.address for s in Sensor.objects.all()} + self.cache = {s.address: s for s in Sensor.objects.all()} diff --git a/greedybear/migrations/0034_remove_unused_log4pot.py b/greedybear/migrations/0034_remove_unused_log4pot.py new file mode 100644 index 00000000..90170635 --- /dev/null +++ b/greedybear/migrations/0034_remove_unused_log4pot.py @@ -0,0 +1,40 @@ +""" +Data migration to remove Log4pot from GeneralHoneypot if it has no associated IOCs. + +This migration fixes issue #773 where Log4pot appears as an active honeypot +in the admin interface and dashboard, despite having no data. + +The migration 0030 created Log4pot with active=True unconditionally, even for +instances that never had Log4pot running. This migration cleans that up by +removing the honeypot entry if it has no IOC data. + +If a user enables Log4Pot on their T-Pot instance later, the extraction +pipeline will automatically create the GeneralHoneypot entry when it +encounters Log4pot data. +""" + +from django.db import migrations + + +def remove_unused_log4pot(apps, schema_editor): + GeneralHoneypot = apps.get_model("greedybear", "GeneralHoneypot") + IOC = apps.get_model("greedybear", "IOC") + + try: + hp = GeneralHoneypot.objects.get(name="Log4pot") + if not IOC.objects.filter(general_honeypot=hp).exists(): + hp.delete() + except GeneralHoneypot.DoesNotExist: + # If the Log4pot honeypot does not exist, there is nothing to clean up. + pass + + +class Migration(migrations.Migration): + + dependencies = [ + ("greedybear", "0033_disable_additional_honeypots"), + ] + + operations = [ + migrations.RunPython(remove_unused_log4pot, reverse_code=migrations.RunPython.noop), + ] diff --git a/greedybear/migrations/0035_rename_greedybear_ip_addr_tor_idx_greedybear__ip_addr_6bc095_idx.py b/greedybear/migrations/0035_rename_greedybear_ip_addr_tor_idx_greedybear__ip_addr_6bc095_idx.py new file mode 100644 index 00000000..bc6d8b04 --- /dev/null +++ b/greedybear/migrations/0035_rename_greedybear_ip_addr_tor_idx_greedybear__ip_addr_6bc095_idx.py @@ -0,0 +1,18 @@ +# Generated by Django 5.2.11 on 2026-02-05 21:18 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('greedybear', '0034_remove_unused_log4pot'), + ] + + operations = [ + migrations.RenameIndex( + model_name='torexitnode', + new_name='greedybear__ip_addr_6bc095_idx', + old_name='greedybear_ip_addr_tor_idx', + ), + ] diff --git a/greedybear/migrations/0036_add_sensors_to_ioc.py b/greedybear/migrations/0036_add_sensors_to_ioc.py new file mode 100644 index 00000000..09ed5b73 --- /dev/null +++ b/greedybear/migrations/0036_add_sensors_to_ioc.py @@ -0,0 +1,58 @@ +# Generated by Django 5.2.11 on 2026-02-09 18:37 + +from django.db import migrations, models + + + +def deduplicate_sensors(apps, schema_editor): + """ + Remove duplicate Sensor records directly from the database to facilitate + the unique constraint on the 'address' field. + For each address with multiple entries, we keep the one with the lowest ID + and remove the others. + """ + Sensor = apps.get_model("greedybear", "Sensor") + db_alias = schema_editor.connection.alias + + # Identify duplicate addresses + # Grouping by address: + from django.db.models import Count + + duplicates = ( + Sensor.objects.using(db_alias) + .values("address") + .annotate(count=Count("id")) + .filter(count__gt=1) + ) + + for entry in duplicates: + address = entry["address"] + # Get all sensors for this address, ordered by ID (oldest first) + sensors = Sensor.objects.using(db_alias).filter(address=address).order_by("id") + + # Keep the first one (lowest ID), delete the rest + first_sensor = sensors.first() + if first_sensor: + # Delete all other sensors with this address + Sensor.objects.using(db_alias).filter(address=address).exclude(id=first_sensor.id).delete() + + +class Migration(migrations.Migration): + + dependencies = [ + ('greedybear', '0035_rename_greedybear_ip_addr_tor_idx_greedybear__ip_addr_6bc095_idx'), + ] + + operations = [ + migrations.AddField( + model_name='ioc', + name='sensors', + field=models.ManyToManyField(blank=True, to='greedybear.sensor'), + ), + migrations.RunPython(deduplicate_sensors, reverse_code=migrations.RunPython.noop), + migrations.AlterField( + model_name='sensor', + name='address', + field=models.CharField(max_length=15, unique=True), + ), + ] diff --git a/greedybear/models.py b/greedybear/models.py index 9aee1fdf..c2a8b316 100644 --- a/greedybear/models.py +++ b/greedybear/models.py @@ -20,7 +20,7 @@ class IocType(models.TextChoices): class Sensor(models.Model): - address = models.CharField(max_length=15, blank=False) + address = models.CharField(max_length=15, blank=False, unique=True) def __str__(self): return self.address @@ -62,6 +62,8 @@ class IOC(models.Model): interaction_count = models.IntegerField(default=1) # FEEDS - list of honeypots from general list, from which the IOC was detected general_honeypot = models.ManyToManyField(GeneralHoneypot, blank=True) + # SENSORS - list of T-Pot sensors that detected this IOC + sensors = models.ManyToManyField(Sensor, blank=True) scanner = models.BooleanField(blank=False, default=False) payload_request = models.BooleanField(blank=False, default=False) related_ioc = models.ManyToManyField("self", blank=True, symmetrical=True) diff --git a/requirements/project-requirements.txt b/requirements/project-requirements.txt index c1fb7466..a8e9cad4 100644 --- a/requirements/project-requirements.txt +++ b/requirements/project-requirements.txt @@ -1,12 +1,12 @@ celery==5.6.2 # if you change this, update the documentation -elasticsearch==9.2.1 +elasticsearch==9.3.0 Django==5.2.11 djangorestframework==3.16.1 django-rest-email-auth==5.0.0 -django-ses==4.6.0 +django-ses==4.7.1 psycopg2-binary==2.9.11 @@ -21,3 +21,5 @@ pandas==3.0.0 scikit-learn==1.8.0 numpy==2.4.2 datasketch==1.9.0 + +feedparser==6.0.12 \ No newline at end of file diff --git a/tests/__init__.py b/tests/__init__.py index 6109d8c3..2dfbc72f 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -182,19 +182,12 @@ def setUpTestData(cls): except User.DoesNotExist: cls.regular_user = User.objects.create_user(username="regular", email="regular@greedybear.com", password="regular") - @classmethod - def tearDownClass(cls): - # db clean - GeneralHoneypot.objects.all().delete() - IOC.objects.all().delete() - CowrieSession.objects.all().delete() - CommandSequence.objects.all().delete() - class ExtractionTestCase(CustomTestCase): def setUp(self): self.mock_ioc_repo = Mock() self.mock_sensor_repo = Mock() + self.mock_sensor_repo.cache = {} # Initialize cache as empty dict for sensor filtering self.mock_session_repo = Mock() def _create_mock_ioc( diff --git a/tests/api/__init__.py b/tests/api/__init__.py new file mode 100644 index 00000000..a14b216c --- /dev/null +++ b/tests/api/__init__.py @@ -0,0 +1 @@ +# This file makes the directory a Python package for test discovery diff --git a/tests/api/views/__init__.py b/tests/api/views/__init__.py new file mode 100644 index 00000000..a14b216c --- /dev/null +++ b/tests/api/views/__init__.py @@ -0,0 +1 @@ +# This file makes the directory a Python package for test discovery diff --git a/tests/api/views/test_command_sequence_view.py b/tests/api/views/test_command_sequence_view.py new file mode 100644 index 00000000..1abcc96b --- /dev/null +++ b/tests/api/views/test_command_sequence_view.py @@ -0,0 +1,91 @@ +from django.test import override_settings +from rest_framework.test import APIClient + +from tests import CustomTestCase + + +class CommandSequenceViewTestCase(CustomTestCase): + """Test cases for the command_sequence_view.""" + + def setUp(self): + # setup client + self.client = APIClient() + self.client.force_authenticate(user=self.superuser) + + def test_missing_query_parameter(self): + """Test that view returns BadRequest when query parameter is missing.""" + response = self.client.get("/api/command_sequence") + self.assertEqual(response.status_code, 400) + + def test_invalid_query_parameter(self): + """Test that view returns BadRequest when query parameter is invalid.""" + response = self.client.get("/api/command_sequence?query=invalid-input}") + self.assertEqual(response.status_code, 400) + + def test_ip_address_query(self): + """Test view with a valid IP address query.""" + response = self.client.get("/api/command_sequence?query=140.246.171.141") + self.assertEqual(response.status_code, 200) + self.assertIn("executed_commands", response.data) + self.assertIn("executed_by", response.data) + + def test_ip_address_query_with_similar(self): + """Test view with a valid IP address query including similar sequences.""" + response = self.client.get("/api/command_sequence?query=140.246.171.141&include_similar") + self.assertEqual(response.status_code, 200) + self.assertIn("executed_commands", response.data) + self.assertIn("executed_by", response.data) + + def test_nonexistent_ip_address(self): + """Test that view returns 404 for IP with no sequences.""" + response = self.client.get("/api/command_sequence?query=10.0.0.1") + self.assertEqual(response.status_code, 404) + + def test_hash_query(self): + """Test view with a valid hash query.""" + response = self.client.get(f"/api/command_sequence?query={self.hash}") + self.assertEqual(response.status_code, 200) + self.assertIn("commands", response.data) + self.assertIn("iocs", response.data) + + def test_hash_query_with_similar(self): + """Test view with a valid hash query including similar sequences.""" + response = self.client.get(f"/api/command_sequence?query={self.hash}&include_similar") + self.assertEqual(response.status_code, 200) + self.assertIn("commands", response.data) + self.assertIn("iocs", response.data) + + def test_nonexistent_hash(self): + """Test that view returns 404 for nonexistent hash.""" + response = self.client.get(f"/api/command_sequence?query={'f' * 64}") + self.assertEqual(response.status_code, 404) + + @override_settings(FEEDS_LICENSE="https://example.com/license") + def test_ip_address_query_with_license(self): + """Test that license is included when FEEDS_LICENSE is populated.""" + response = self.client.get("/api/command_sequence?query=140.246.171.141") + self.assertEqual(response.status_code, 200) + self.assertIn("license", response.data) + self.assertEqual(response.data["license"], "https://example.com/license") + + @override_settings(FEEDS_LICENSE="") + def test_ip_address_query_without_license(self): + """Test that license is not included when FEEDS_LICENSE is empty.""" + response = self.client.get("/api/command_sequence?query=140.246.171.141") + self.assertEqual(response.status_code, 200) + self.assertNotIn("license", response.data) + + @override_settings(FEEDS_LICENSE="https://example.com/license") + def test_hash_query_with_license(self): + """Test that license is included when FEEDS_LICENSE is populated.""" + response = self.client.get(f"/api/command_sequence?query={self.hash}") + self.assertEqual(response.status_code, 200) + self.assertIn("license", response.data) + self.assertEqual(response.data["license"], "https://example.com/license") + + @override_settings(FEEDS_LICENSE="") + def test_hash_query_without_license(self): + """Test that license is not included when FEEDS_LICENSE is empty.""" + response = self.client.get(f"/api/command_sequence?query={self.hash}") + self.assertEqual(response.status_code, 200) + self.assertNotIn("license", response.data) diff --git a/tests/api/views/test_cowrie_session_view.py b/tests/api/views/test_cowrie_session_view.py new file mode 100644 index 00000000..e7d01bed --- /dev/null +++ b/tests/api/views/test_cowrie_session_view.py @@ -0,0 +1,224 @@ +from django.test import override_settings +from rest_framework.test import APIClient + +from tests import CustomTestCase + + +class CowrieSessionViewTestCase(CustomTestCase): + """Test cases for the cowrie_session_view.""" + + def setUp(self): + # setup client + self.client = APIClient() + self.client.force_authenticate(user=self.superuser) + + # # # # # Basic IP Query Test # # # # # + def test_ip_address_query(self): + """Test view with a valid IP address query.""" + response = self.client.get("/api/cowrie_session?query=140.246.171.141") + self.assertEqual(response.status_code, 200) + self.assertIn("query", response.data) + self.assertIn("commands", response.data) + self.assertIn("sources", response.data) + self.assertNotIn("credentials", response.data) + self.assertNotIn("sessions", response.data) + + def test_ip_address_query_with_similar(self): + """Test view with a valid IP address query including similar sequences.""" + response = self.client.get("/api/cowrie_session?query=140.246.171.141&include_similar=true") + self.assertEqual(response.status_code, 200) + self.assertIn("query", response.data) + self.assertIn("commands", response.data) + self.assertIn("sources", response.data) + self.assertNotIn("credentials", response.data) + self.assertNotIn("sessions", response.data) + self.assertEqual(len(response.data["sources"]), 2) + + def test_ip_address_query_with_credentials(self): + """Test view with a valid IP address query including credentials.""" + response = self.client.get("/api/cowrie_session?query=140.246.171.141&include_credentials=true") + self.assertEqual(response.status_code, 200) + self.assertIn("query", response.data) + self.assertIn("commands", response.data) + self.assertIn("sources", response.data) + self.assertIn("credentials", response.data) + self.assertNotIn("sessions", response.data) + self.assertEqual(len(response.data["credentials"]), 1) + self.assertEqual(response.data["credentials"][0], "root | root") + + def test_ip_address_query_with_sessions(self): + """Test view with a valid IP address query including session data.""" + response = self.client.get("/api/cowrie_session?query=140.246.171.141&include_session_data=true") + self.assertEqual(response.status_code, 200) + self.assertIn("query", response.data) + self.assertIn("commands", response.data) + self.assertIn("sources", response.data) + self.assertNotIn("credentials", response.data) + self.assertIn("sessions", response.data) + self.assertEqual(len(response.data["sessions"]), 1) + self.assertIn("time", response.data["sessions"][0]) + self.assertEqual(response.data["sessions"][0]["duration"], 1.234) + self.assertEqual(response.data["sessions"][0]["source"], "140.246.171.141") + self.assertEqual(response.data["sessions"][0]["interactions"], 5) + self.assertEqual(response.data["sessions"][0]["credentials"][0], "root | root") + self.assertEqual(response.data["sessions"][0]["commands"], "cd foo\nls -la") + + def test_ip_address_query_with_all(self): + """Test view with a valid IP address query including everything.""" + response = self.client.get("/api/cowrie_session?query=140.246.171.141&include_similar=true&include_credentials=true&include_session_data=true") + self.assertEqual(response.status_code, 200) + self.assertIn("query", response.data) + self.assertIn("commands", response.data) + self.assertIn("sources", response.data) + self.assertIn("credentials", response.data) + self.assertIn("sessions", response.data) + + # # # # # Basic Hash Query Test # # # # # + def test_hash_query(self): + """Test view with a valid hash query.""" + response = self.client.get(f"/api/cowrie_session?query={self.hash}") + self.assertEqual(response.status_code, 200) + self.assertIn("query", response.data) + self.assertIn("commands", response.data) + self.assertIn("sources", response.data) + self.assertNotIn("credentials", response.data) + self.assertNotIn("sessions", response.data) + + def test_hash_query_with_all(self): + """Test view with a valid hash query including everything.""" + response = self.client.get(f"/api/cowrie_session?query={self.hash}&include_similar=true&include_credentials=true&include_session_data=true") + self.assertEqual(response.status_code, 200) + self.assertIn("query", response.data) + self.assertIn("commands", response.data) + self.assertIn("sources", response.data) + self.assertIn("credentials", response.data) + self.assertIn("sessions", response.data) + self.assertEqual(len(response.data["sources"]), 2) + + # # # # # IP Address Validation Tests # # # # # + def test_nonexistent_ip_address(self): + """Test that view returns 404 for IP with no sequences.""" + response = self.client.get("/api/cowrie_session?query=10.0.0.1") + self.assertEqual(response.status_code, 404) + + def test_ipv6_address_query(self): + """Test view with a valid IPv6 address query.""" + response = self.client.get("/api/cowrie_session?query=2001:db8::1") + self.assertEqual(response.status_code, 404) + + def test_invalid_ip_format(self): + """Test that malformed IP addresses are rejected.""" + response = self.client.get("/api/cowrie_session?query=999.999.999.999") + self.assertEqual(response.status_code, 400) + + def test_ip_with_cidr_notation(self): + """Test that CIDR notation is rejected.""" + response = self.client.get("/api/cowrie_session?query=192.168.1.0/24") + self.assertEqual(response.status_code, 400) + + # # # # # Parameter Validation Tests # # # # # + def test_missing_query_parameter(self): + """Test that view returns BadRequest when query parameter is missing.""" + response = self.client.get("/api/cowrie_session") + self.assertEqual(response.status_code, 400) + + def test_invalid_query_parameter(self): + """Test that view returns BadRequest when query parameter is invalid.""" + response = self.client.get("/api/cowrie_session?query=invalid-input}") + self.assertEqual(response.status_code, 400) + + def test_include_credentials_invalid_value(self): + """Test that invalid boolean values default to false.""" + response = self.client.get("/api/cowrie_session?query=140.246.171.141&include_credentials=maybe") + self.assertEqual(response.status_code, 200) + self.assertNotIn("credentials", response.data) + + def test_case_insensitive_boolean_parameters(self): + """Test that boolean parameters accept various case formats.""" + response = self.client.get("/api/cowrie_session?query=140.246.171.141&include_credentials=TRUE") + self.assertEqual(response.status_code, 200) + self.assertIn("credentials", response.data) + + response = self.client.get("/api/cowrie_session?query=140.246.171.141&include_credentials=True") + self.assertEqual(response.status_code, 200) + self.assertIn("credentials", response.data) + + # # # # # Hash Validation Tests # # # # # + def test_nonexistent_hash(self): + """Test that view returns 404 for nonexistent hash.""" + response = self.client.get(f"/api/cowrie_session?query={'f' * 64}") + self.assertEqual(response.status_code, 404) + + def test_hash_wrong_length(self): + """Test that hashes with incorrect length are rejected.""" + response = self.client.get("/api/cowrie_session?query=" + "a" * 32) # 32 chars instead of 64 + self.assertEqual(response.status_code, 400) + + def test_hash_invalid_characters(self): + """Test that hashes with invalid characters are rejected.""" + invalid_hash = "g" * 64 # 'g' is not a valid hex character + response = self.client.get(f"/api/cowrie_session?query={invalid_hash}") + self.assertEqual(response.status_code, 400) + + def test_hash_case_insensitive(self): + """Test that hash queries are case-insensitive.""" + response_lower = self.client.get(f"/api/cowrie_session?query={self.hash.lower()}") + response_upper = self.client.get(f"/api/cowrie_session?query={self.hash.upper()}") + self.assertEqual(response_lower.status_code, response_upper.status_code) + + # # # # # Special Characters & Encoding Tests # # # # # + def test_query_with_url_encoding(self): + """Test that URL-encoded queries work correctly.""" + response = self.client.get("/api/cowrie_session?query=140.246.171.141%20") + # Should either work or return 400, not crash + self.assertIn(response.status_code, [200, 400, 404]) + + # # # # # License Tests # # # # # + @override_settings(FEEDS_LICENSE="https://example.com/license") + def test_ip_query_with_license(self): + """Test that license is included when FEEDS_LICENSE is populated.""" + response = self.client.get("/api/cowrie_session?query=140.246.171.141") + self.assertEqual(response.status_code, 200) + self.assertIn("license", response.data) + self.assertEqual(response.data["license"], "https://example.com/license") + + @override_settings(FEEDS_LICENSE="") + def test_ip_query_without_license(self): + """Test that license is not included when FEEDS_LICENSE is empty.""" + response = self.client.get("/api/cowrie_session?query=140.246.171.141") + self.assertEqual(response.status_code, 200) + self.assertNotIn("license", response.data) + + @override_settings(FEEDS_LICENSE="https://example.com/license") + def test_hash_query_with_license(self): + """Test that license is included when FEEDS_LICENSE is populated.""" + response = self.client.get(f"/api/cowrie_session?query={self.hash}") + self.assertEqual(response.status_code, 200) + self.assertIn("license", response.data) + self.assertEqual(response.data["license"], "https://example.com/license") + + @override_settings(FEEDS_LICENSE="") + def test_hash_query_without_license(self): + """Test that license is not included when FEEDS_LICENSE is empty.""" + response = self.client.get(f"/api/cowrie_session?query={self.hash}") + self.assertEqual(response.status_code, 200) + self.assertNotIn("license", response.data) + + def test_query_with_special_characters(self): + """Test handling of queries with special characters.""" + response = self.client.get("/api/cowrie_session?query=") + self.assertEqual(response.status_code, 400) + + # # # # # Authentication & Authorization Tests # # # # # + def test_unauthenticated_request(self): + """Test that unauthenticated requests are rejected.""" + client = APIClient() # No authentication + response = client.get("/api/cowrie_session?query=140.246.171.141") + self.assertEqual(response.status_code, 401) + + def test_regular_user_access(self): + """Test that regular (non-superuser) authenticated users can access.""" + client = APIClient() + client.force_authenticate(user=self.regular_user) + response = client.get("/api/cowrie_session?query=140.246.171.141") + self.assertEqual(response.status_code, 200) diff --git a/tests/api/views/test_enrichment_view.py b/tests/api/views/test_enrichment_view.py new file mode 100644 index 00000000..47653f32 --- /dev/null +++ b/tests/api/views/test_enrichment_view.py @@ -0,0 +1,61 @@ +from rest_framework.test import APIClient + +from tests import CustomTestCase + + +class EnrichmentViewTestCase(CustomTestCase): + def setUp(self): + # setup client + self.client = APIClient() + self.client.force_authenticate(user=self.superuser) + + def test_for_vaild_unregistered_ip(self): + """Check for a valid IP that is unavaliable in DB""" + response = self.client.get("/api/enrichment?query=192.168.0.1") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()["found"], False) + + def test_for_invaild_unregistered_ip(self): + """Check for a IP that Fails Regex Checks and is unavaliable in DB""" + response = self.client.get("/api/enrichment?query=30.168.1.255.1") + self.assertEqual(response.status_code, 400) + + def test_for_vaild_registered_ip(self): + """Check for a valid IP that is avaliable in DB""" + response = self.client.get("/api/enrichment?query=140.246.171.141") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()["found"], True) + self.assertEqual(response.json()["ioc"]["name"], self.ioc.name) + self.assertEqual(response.json()["ioc"]["type"], self.ioc.type) + self.assertEqual( + response.json()["ioc"]["first_seen"], + self.ioc.first_seen.isoformat(sep="T", timespec="microseconds"), + ) + self.assertEqual( + response.json()["ioc"]["last_seen"], + self.ioc.last_seen.isoformat(sep="T", timespec="microseconds"), + ) + self.assertEqual(response.json()["ioc"]["number_of_days_seen"], self.ioc.number_of_days_seen) + self.assertEqual(response.json()["ioc"]["attack_count"], self.ioc.attack_count) + # Honeypots are now via M2M relationship (serialized as list of strings) + honeypot_names = response.json()["ioc"]["general_honeypot"] + self.assertIn(self.heralding.name, honeypot_names) + self.assertIn(self.ciscoasa.name, honeypot_names) + self.assertIn(self.cowrie_hp.name, honeypot_names) + self.assertIn(self.log4pot_hp.name, honeypot_names) + self.assertEqual(response.json()["ioc"]["scanner"], self.ioc.scanner) + self.assertEqual(response.json()["ioc"]["payload_request"], self.ioc.payload_request) + self.assertEqual( + response.json()["ioc"]["recurrence_probability"], + self.ioc.recurrence_probability, + ) + self.assertEqual( + response.json()["ioc"]["expected_interactions"], + self.ioc.expected_interactions, + ) + + def test_for_invalid_authentication(self): + """Check for a invalid authentication""" + self.client.logout() + response = self.client.get("/api/enrichment?query=140.246.171.141") + self.assertEqual(response.status_code, 401) diff --git a/tests/api/views/test_feeds_advanced_view.py b/tests/api/views/test_feeds_advanced_view.py new file mode 100644 index 00000000..20ea7072 --- /dev/null +++ b/tests/api/views/test_feeds_advanced_view.py @@ -0,0 +1,80 @@ +from django.conf import settings +from rest_framework.test import APIClient + +from tests import CustomTestCase + + +class FeedsAdvancedViewTestCase(CustomTestCase): + def setUp(self): + self.client = APIClient() + self.client.force_authenticate(user=self.superuser) + + def test_200_all_feeds(self): + response = self.client.get("/api/feeds/advanced/") + self.assertEqual(response.status_code, 200) + if settings.FEEDS_LICENSE: + self.assertEqual(response.json()["license"], settings.FEEDS_LICENSE) + else: + self.assertNotIn("license", response.json()) + + iocs = response.json()["iocs"] + target_ioc = next((i for i in iocs if i["value"] == self.ioc.name), None) + self.assertIsNotNone(target_ioc) + + self.assertEqual(set(target_ioc["feed_type"]), {"log4pot", "cowrie", "heralding", "ciscoasa"}) + self.assertEqual(target_ioc["attack_count"], 1) + self.assertEqual(target_ioc["scanner"], True) + self.assertEqual(target_ioc["payload_request"], True) + self.assertEqual(target_ioc["recurrence_probability"], self.ioc.recurrence_probability) + self.assertEqual(target_ioc["expected_interactions"], self.ioc.expected_interactions) + + def test_200_general_feeds(self): + response = self.client.get("/api/feeds/advanced/?feed_type=heralding") + self.assertEqual(response.status_code, 200) + if settings.FEEDS_LICENSE: + self.assertEqual(response.json()["license"], settings.FEEDS_LICENSE) + else: + self.assertNotIn("license", response.json()) + + iocs = response.json()["iocs"] + target_ioc = next((i for i in iocs if i["value"] == self.ioc.name), None) + self.assertIsNotNone(target_ioc) + + self.assertEqual(set(target_ioc["feed_type"]), {"log4pot", "cowrie", "heralding", "ciscoasa"}) + self.assertEqual(target_ioc["attack_count"], 1) + self.assertEqual(target_ioc["scanner"], True) + self.assertEqual(target_ioc["payload_request"], True) + self.assertEqual(target_ioc["recurrence_probability"], self.ioc.recurrence_probability) + self.assertEqual(target_ioc["expected_interactions"], self.ioc.expected_interactions) + + def test_400_feeds(self): + response = self.client.get("/api/feeds/advanced/?attack_type=test") + self.assertEqual(response.status_code, 400) + + def test_200_feeds_pagination(self): + response = self.client.get("/api/feeds/advanced/?paginate=true&page_size=10&page=1") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()["count"], 4) + self.assertEqual(response.json()["total_pages"], 1) + + def test_200_feeds_pagination_include(self): + response = self.client.get("/api/feeds/advanced/?paginate=true&page_size=10&page=1&include_reputation=mass%20scanner") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()["count"], 1) + self.assertEqual(response.json()["total_pages"], 1) + + def test_200_feeds_pagination_exclude_mass(self): + response = self.client.get("/api/feeds/advanced/?paginate=true&page_size=10&page=1&exclude_reputation=mass%20scanner") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()["count"], 3) + self.assertEqual(response.json()["total_pages"], 1) + + def test_200_feeds_pagination_exclude_tor(self): + response = self.client.get("/api/feeds/advanced/?paginate=true&page_size=10&page=1&exclude_reputation=tor%20exit%20node") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()["count"], 3) + self.assertEqual(response.json()["total_pages"], 1) + + def test_400_feeds_pagination(self): + response = self.client.get("/api/feeds/advanced/?paginate=true&page_size=10&page=1&attack_type=test") + self.assertEqual(response.status_code, 400) diff --git a/tests/api/views/test_feeds_asn_view.py b/tests/api/views/test_feeds_asn_view.py new file mode 100644 index 00000000..68e10244 --- /dev/null +++ b/tests/api/views/test_feeds_asn_view.py @@ -0,0 +1,184 @@ +from django.utils import timezone +from rest_framework.test import APIClient + +from greedybear.models import IOC, GeneralHoneypot +from tests import CustomTestCase + + +class FeedsASNViewTestCase(CustomTestCase): + """Tests for ASN aggregated feeds API""" + + @classmethod + def setUpClass(cls): + super().setUpClass() + IOC.objects.all().delete() + cls.testpot1, _ = GeneralHoneypot.objects.get_or_create(name="testpot1", active=True) + cls.testpot2, _ = GeneralHoneypot.objects.get_or_create(name="testpot2", active=True) + + cls.high_asn = "13335" + cls.low_asn = "16276" + + cls.ioc_high1 = IOC.objects.create( + name="high1.example.com", + type="ip", + asn=cls.high_asn, + attack_count=15, + interaction_count=30, + login_attempts=5, + first_seen=timezone.now() - timezone.timedelta(days=10), + recurrence_probability=0.8, + expected_interactions=20.0, + ) + cls.ioc_high1.general_honeypot.add(cls.testpot1, cls.testpot2) + cls.ioc_high1.save() + + cls.ioc_high2 = IOC.objects.create( + name="high2.example.com", + type="ip", + asn=cls.high_asn, + attack_count=5, + interaction_count=10, + login_attempts=2, + first_seen=timezone.now() - timezone.timedelta(days=5), + recurrence_probability=0.3, + expected_interactions=8.0, + ) + cls.ioc_high2.general_honeypot.add(cls.testpot1, cls.testpot2) + cls.ioc_high2.save() + + cls.ioc_low = IOC.objects.create( + name="low.example.com", + type="ip", + asn=cls.low_asn, + attack_count=2, + interaction_count=5, + login_attempts=1, + first_seen=timezone.now(), + recurrence_probability=0.1, + expected_interactions=3.0, + ) + cls.ioc_low.general_honeypot.add(cls.testpot1, cls.testpot2) + cls.ioc_low.save() + + def setUp(self): + self.client = APIClient() + self.client.force_authenticate(user=self.superuser) + self.url = "/api/feeds/asn/" + + def _get_results(self, response): + payload = response.json() + self.assertIsInstance(payload, list) + return payload + + def test_200_asn_feed_aggregated_fields(self): + """Ensure aggregated fields are computed correctly per ASN using dynamic sums""" + response = self.client.get(self.url) + self.assertEqual(response.status_code, 200) + results = self._get_results(response) + + # filtering high ASN + high_item = next((item for item in results if str(item["asn"]) == self.high_asn), None) + self.assertIsNotNone(high_item) + + # getting all IOCs for high ASN from the DB + high_iocs = IOC.objects.filter(asn=self.high_asn) + + self.assertEqual(high_item["ioc_count"], high_iocs.count()) + self.assertEqual(high_item["total_attack_count"], sum(i.attack_count for i in high_iocs)) + self.assertEqual(high_item["total_interaction_count"], sum(i.interaction_count for i in high_iocs)) + self.assertEqual(high_item["total_login_attempts"], sum(i.login_attempts for i in high_iocs)) + self.assertAlmostEqual(high_item["expected_ioc_count"], sum(i.recurrence_probability for i in high_iocs)) + self.assertAlmostEqual(high_item["expected_interactions"], sum(i.expected_interactions for i in high_iocs)) + + # validating first_seen / last_seen dynamically + self.assertEqual(high_item["first_seen"], min(i.first_seen for i in high_iocs).isoformat()) + self.assertEqual(high_item["last_seen"], max(i.last_seen for i in high_iocs).isoformat()) + + # validating honeypots dynamically + expected_honeypots = sorted({hp.name for i in high_iocs for hp in i.general_honeypot.all()}) + self.assertEqual(sorted(high_item["honeypots"]), expected_honeypots) + + def test_200_asn_feed_default_ordering(self): + response = self.client.get(self.url) + self.assertEqual(response.status_code, 200) + results = self._get_results(response) + + # high_asn has ioc_count=2 > low_asn ioc_count=1 + self.assertEqual(str(results[0]["asn"]), self.high_asn) + self.assertEqual(str(results[1]["asn"]), self.low_asn) + + def test_200_asn_feed_ordering_desc_ioc_count(self): + response = self.client.get(self.url + "?ordering=-ioc_count") + self.assertEqual(response.status_code, 200) + results = self._get_results(response) + + self.assertEqual(str(results[0]["asn"]), self.high_asn) + + def test_200_asn_feed_ordering_asc_ioc_count(self): + response = self.client.get(self.url + "?ordering=ioc_count") + self.assertEqual(response.status_code, 200) + results = self._get_results(response) + self.assertEqual(str(results[0]["asn"]), self.low_asn) + + def test_200_asn_feed_ordering_desc_interaction_count(self): + response = self.client.get(self.url + "?ordering=-total_interaction_count") + self.assertEqual(response.status_code, 200) + results = self._get_results(response) + self.assertEqual(str(results[0]["asn"]), self.high_asn) + + def test_200_asn_feed_with_asn_filter(self): + response = self.client.get(self.url + f"?asn={self.high_asn}") + self.assertEqual(response.status_code, 200) + + results = self._get_results(response) + self.assertEqual(len(results), 1) + self.assertEqual(str(results[0]["asn"]), self.high_asn) + + def test_400_asn_feed_invalid_ordering_honeypots(self): + response = self.client.get(self.url + "?ordering=honeypots") + self.assertEqual(response.status_code, 400) + data = response.json() + errors_container = data.get("errors", data) + error_list = errors_container.get("ordering", []) + self.assertTrue(error_list) + error_msg = error_list[0].lower() + self.assertIn("honeypots", error_msg) + self.assertIn("invalid", error_msg) + + def test_400_asn_feed_invalid_ordering_random(self): + response = self.client.get(self.url + "?ordering=xyz123") + self.assertEqual(response.status_code, 400) + data = response.json() + errors_container = data.get("errors", data) + error_list = errors_container.get("ordering", []) + self.assertTrue(error_list) + error_msg = error_list[0].lower() + self.assertIn("xyz123", error_msg) + self.assertIn("invalid", error_msg) + + def test_400_asn_feed_invalid_ordering_model_field_not_in_agg(self): + response = self.client.get(self.url + "?ordering=attack_count") + self.assertEqual(response.status_code, 400) + data = response.json() + errors_container = data.get("errors", data) + error_list = errors_container.get("ordering", []) + self.assertTrue(error_list) + error_msg = error_list[0].lower() + self.assertIn("attack_count", error_msg) + self.assertIn("invalid", error_msg) + + def test_400_asn_feed_ordering_empty_param(self): + response = self.client.get(self.url + "?ordering=") + self.assertEqual(response.status_code, 400) + data = response.json() + errors_container = data.get("errors", data) + error_list = errors_container.get("ordering", []) + self.assertTrue(error_list) + error_msg = error_list[0].lower() + self.assertIn("blank", error_msg) + + def test_asn_feed_ignores_feed_size(self): + response = self.client.get(self.url + "?feed_size=1") + results = response.json() + # aggregation should return all ASNs regardless of feed_size + self.assertEqual(len(results), 2) diff --git a/tests/api/views/test_feeds_view.py b/tests/api/views/test_feeds_view.py new file mode 100644 index 00000000..5916d8dc --- /dev/null +++ b/tests/api/views/test_feeds_view.py @@ -0,0 +1,133 @@ +from django.conf import settings +from django.test import override_settings + +from tests import CustomTestCase + + +class FeedsViewTestCase(CustomTestCase): + def test_200_log4pot_feeds(self): + response = self.client.get("/api/feeds/log4pot/all/recent.json") + self.assertEqual(response.status_code, 200) + if settings.FEEDS_LICENSE: + self.assertEqual(response.json()["license"], settings.FEEDS_LICENSE) + else: + self.assertNotIn("license", response.json()) + + iocs = response.json()["iocs"] + target_ioc = next((i for i in iocs if i["value"] == self.ioc.name), None) + self.assertIsNotNone(target_ioc) + + # feed_type now derived from general_honeypot M2M + self.assertIn("log4pot", target_ioc["feed_type"]) + self.assertIn("cowrie", target_ioc["feed_type"]) + self.assertIn("heralding", target_ioc["feed_type"]) + self.assertIn("ciscoasa", target_ioc["feed_type"]) + self.assertEqual(target_ioc["attack_count"], 1) + self.assertEqual(target_ioc["scanner"], True) + self.assertEqual(target_ioc["payload_request"], True) + self.assertEqual(target_ioc["recurrence_probability"], self.ioc.recurrence_probability) + self.assertEqual(target_ioc["expected_interactions"], self.ioc.expected_interactions) + + @override_settings(FEEDS_LICENSE="https://example.com/license") + def test_200_all_feeds_with_license(self): + """Test feeds endpoint when FEEDS_LICENSE is populated""" + response = self.client.get("/api/feeds/all/all/recent.json") + self.assertEqual(response.status_code, 200) + self.assertIn("license", response.json()) + self.assertEqual(response.json()["license"], "https://example.com/license") + + @override_settings(FEEDS_LICENSE="") + def test_200_all_feeds_without_license(self): + """Test feeds endpoint when FEEDS_LICENSE is empty""" + response = self.client.get("/api/feeds/all/all/recent.json") + self.assertEqual(response.status_code, 200) + self.assertNotIn("license", response.json()) + + def test_200_general_feeds(self): + response = self.client.get("/api/feeds/heralding/all/recent.json") + self.assertEqual(response.status_code, 200) + if settings.FEEDS_LICENSE: + self.assertEqual(response.json()["license"], settings.FEEDS_LICENSE) + else: + self.assertNotIn("license", response.json()) + + iocs = response.json()["iocs"] + target_ioc = next((i for i in iocs if i["value"] == self.ioc.name), None) + self.assertIsNotNone(target_ioc) + + self.assertEqual(set(target_ioc["feed_type"]), {"log4pot", "cowrie", "heralding", "ciscoasa"}) + self.assertEqual(target_ioc["attack_count"], 1) + self.assertEqual(target_ioc["scanner"], True) + self.assertEqual(target_ioc["payload_request"], True) + self.assertEqual(target_ioc["recurrence_probability"], self.ioc.recurrence_probability) + self.assertEqual(target_ioc["expected_interactions"], self.ioc.expected_interactions) + + def test_200_feeds_scanner_inclusion(self): + response = self.client.get("/api/feeds/heralding/all/recent.json?include_mass_scanners") + self.assertEqual(response.status_code, 200) + if settings.FEEDS_LICENSE: + self.assertEqual(response.json()["license"], settings.FEEDS_LICENSE) + else: + self.assertNotIn("license", response.json()) + # Expecting 3 because setupTestData creates 3 IOCs (ioc, ioc_2, ioc_domain) associated with Heralding + self.assertEqual(len(response.json()["iocs"]), 3) + + def test_400_feeds(self): + response = self.client.get("/api/feeds/test/all/recent.json") + self.assertEqual(response.status_code, 400) + + def test_200_feeds_pagination(self): + response = self.client.get("/api/feeds/?page_size=10&page=1&feed_type=all&attack_type=all&age=recent") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()["count"], 2) + self.assertEqual(response.json()["total_pages"], 1) + + def test_200_feeds_pagination_inclusion_mass(self): + response = self.client.get("/api/feeds/?page_size=10&page=1&feed_type=all&attack_type=all&age=recent&include_mass_scanners") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()["count"], 3) + + def test_200_feeds_pagination_inclusion_tor(self): + response = self.client.get("/api/feeds/?page_size=10&page=1&feed_type=all&attack_type=all&age=recent&include_tor_exit_nodes") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()["count"], 3) + + def test_200_feeds_pagination_inclusion_mass_and_tor(self): + response = self.client.get("/api/feeds/?page_size=10&page=1&feed_type=all&attack_type=all&age=recent&include_mass_scanners&include_tor_exit_nodes") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()["count"], 4) + + def test_200_feeds_filter_ip_only(self): + response = self.client.get("/api/feeds/all/all/recent.json?ioc_type=ip") + self.assertEqual(response.status_code, 200) + # Should only return IP addresses, not domains + for ioc in response.json()["iocs"]: + # Verify all returned values are IPs (contain dots and numbers pattern) + self.assertRegex(ioc["value"], r"^\d+\.\d+\.\d+\.\d+$") + + def test_200_feeds_filter_domain_only(self): + response = self.client.get("/api/feeds/all/all/recent.json?ioc_type=domain") + self.assertEqual(response.status_code, 200) + # Should only return domains, not IPs + self.assertGreater(len(response.json()["iocs"]), 0) + for ioc in response.json()["iocs"]: + # Verify all returned values are domains (contain alphabetic characters) + self.assertRegex(ioc["value"], r"[a-zA-Z]") + + def test_200_feeds_pagination_filter_ip(self): + response = self.client.get( + "/api/feeds/?page_size=10&page=1&feed_type=all&attack_type=all&age=recent&ioc_type=ip&include_mass_scanners&include_tor_exit_nodes" + ) + self.assertEqual(response.status_code, 200) + # Should return only IPs (3 in test data) + self.assertEqual(response.json()["count"], 3) + + def test_200_feeds_pagination_filter_domain(self): + response = self.client.get("/api/feeds/?page_size=10&page=1&feed_type=all&attack_type=all&age=recent&ioc_type=domain") + self.assertEqual(response.status_code, 200) + # Should return only domains (1 in test data) + self.assertEqual(response.json()["count"], 1) + + def test_400_feeds_pagination(self): + response = self.client.get("/api/feeds/?page_size=10&page=1&feed_type=all&attack_type=test&age=recent") + self.assertEqual(response.status_code, 400) diff --git a/tests/api/views/test_general_honeypot_view.py b/tests/api/views/test_general_honeypot_view.py new file mode 100644 index 00000000..bdc5a5e4 --- /dev/null +++ b/tests/api/views/test_general_honeypot_view.py @@ -0,0 +1,25 @@ +from greedybear.models import GeneralHoneypot +from tests import CustomTestCase + + +class GeneralHoneypotViewTestCase(CustomTestCase): + def test_200_all_general_honeypots(self): + initial_count = GeneralHoneypot.objects.count() + # add a general honeypot not active + GeneralHoneypot(name="Adbhoney", active=False).save() + self.assertEqual(GeneralHoneypot.objects.count(), initial_count + 1) + + response = self.client.get("/api/general_honeypot") + self.assertEqual(response.status_code, 200) + # Verify the newly created honeypot is in the response + self.assertIn("Adbhoney", response.json()) + + def test_200_active_general_honeypots(self): + response = self.client.get("/api/general_honeypot?onlyActive=true") + self.assertEqual(response.status_code, 200) + result = response.json() + # Should include active honeypots from CustomTestCase + self.assertIn("Heralding", result) + self.assertIn("Ciscoasa", result) + # Should NOT include inactive honeypot + self.assertNotIn("Ddospot", result) diff --git a/tests/api/views/test_news_view.py b/tests/api/views/test_news_view.py new file mode 100644 index 00000000..3cb78ca1 --- /dev/null +++ b/tests/api/views/test_news_view.py @@ -0,0 +1,166 @@ +from unittest.mock import patch + +import requests +from django.core.cache import cache +from feedparser import FeedParserDict + +from api.views.utils import CACHE_KEY_GREEDYBEAR_NEWS, get_greedybear_news +from tests import CustomTestCase + + +class NewsTestCase(CustomTestCase): + def setUp(self): + cache.clear() + + def tearDown(self): + cache.clear() + + @patch("api.views.utils.feedparser.parse") + def test_returns_cached_data(self, mock_parse): + cached_data = [ + { + "title": "GreedyBear Cached", + "date": "Thu, 29 Jan 2026 00:00:00 GMT", + "link": "https://example.com", + "subtext": "cached content", + } + ] + cache.set(CACHE_KEY_GREEDYBEAR_NEWS, cached_data, 300) + + result = get_greedybear_news() + + self.assertEqual(result, cached_data) + mock_parse.assert_not_called() + + @patch("api.views.utils.feedparser.parse") + def test_filters_only_greedybear_posts(self, mock_parse): + mock_parse.return_value = FeedParserDict( + entries=[ + FeedParserDict( + title="IntelOwl Update", + summary="intelowl news", + published="Wed, 01 Jan 2026 00:00:00 GMT", + published_parsed=(2026, 1, 1, 0, 0, 0, 2, 1, 0), + link="https://example.com/1", + ), + FeedParserDict( + title="GreedyBear v3 Release", + summary="greedybear release notes", + published="Thu, 29 Jan 2026 00:00:00 GMT", + published_parsed=(2026, 1, 29, 0, 0, 0, 3, 29, 0), + link="https://example.com/2", + ), + FeedParserDict( + title="IntelOwl Improvements", + summary="Not related to GreedyBear", + published="Mon, 01 Sep 2025 00:00:00 GMT", + published_parsed=(2025, 9, 1, 0, 0, 0, 0, 244, 0), + link="https://example.com/3", + ), + ] + ) + + result = get_greedybear_news() + self.assertEqual(len(result), 1) + self.assertEqual(result[0]["title"], "GreedyBear v3 Release") + + @patch("api.views.utils.feedparser.parse") + def test_sorts_posts_by_date_desc(self, mock_parse): + mock_parse.return_value = FeedParserDict( + entries=[ + FeedParserDict( + title="GreedyBear Old", + summary="old post", + published="Wed, 01 Jan 2026 00:00:00 GMT", + published_parsed=(2026, 1, 1, 0, 0, 0, 0, 0, 0), + link="https://example.com/old", + ), + FeedParserDict( + title="GreedyBear New", + summary="new post", + published="Thu, 29 Jan 2026 00:00:00 GMT", + published_parsed=(2026, 1, 29, 0, 0, 0, 0, 0, 0), + link="https://example.com/new", + ), + ] + ) + + result = get_greedybear_news() + self.assertEqual(len(result), 2) + self.assertEqual(result[0]["title"], "GreedyBear New") + self.assertEqual(result[1]["title"], "GreedyBear Old") + + @patch("api.views.utils.feedparser.parse") + def test_truncates_long_summary(self, mock_parse): + long_summary = "word " * 100 + mock_parse.return_value = FeedParserDict( + entries=[ + FeedParserDict( + title="GreedyBear Long Post", + summary=long_summary, + published="Thu, 29 Jan 2026 00:00:00 GMT", + published_parsed=(2026, 1, 29, 0, 0, 0, 3, 29, 0), + link="https://example.com", + ) + ] + ) + + result = get_greedybear_news() + self.assertEqual(len(result), 1) + self.assertTrue(result[0]["subtext"].endswith("...")) + self.assertLessEqual(len(result[0]["subtext"]), 184) + + @patch("api.views.utils.feedparser.parse") + def test_skips_entries_without_published_date(self, mock_parse): + mock_parse.return_value = FeedParserDict( + entries=[ + FeedParserDict( + title="GreedyBear No Date", + summary="missing date", + link="https://example.com", + ) + ] + ) + + result = get_greedybear_news() + self.assertEqual(result, []) + + @patch("api.views.utils.feedparser.parse") + def test_handles_feed_failure_gracefully(self, mock_parse): + mock_parse.side_effect = Exception("Feed error") + result = get_greedybear_news() + self.assertEqual(result, []) + + @patch("api.views.utils.feedparser.parse") + def test_results_are_cached_after_first_call(self, mock_parse): + mock_parse.return_value = FeedParserDict( + entries=[ + FeedParserDict( + title="GreedyBear Cached Test", + summary="cache test", + published="Thu, 29 Jan 2026 00:00:00 GMT", + published_parsed=(2026, 1, 29, 0, 0, 0, 3, 29, 0), + link="https://example.com", + ) + ] + ) + + # first calling hits feed + result1 = get_greedybear_news() + self.assertEqual(len(result1), 1) + + # resetting mock to ensure cache is used + mock_parse.reset_mock() + + # second call should use cache + result2 = get_greedybear_news() + self.assertEqual(result1, result2) + mock_parse.assert_not_called() + + @patch("api.views.utils.requests.get") + def test_feed_request_timeout_returns_empty_list(self, mock_get): + mock_get.side_effect = requests.Timeout() + + result = get_greedybear_news() + + self.assertEqual(result, []) diff --git a/tests/api/views/test_statistics_view.py b/tests/api/views/test_statistics_view.py new file mode 100644 index 00000000..f45f2bb9 --- /dev/null +++ b/tests/api/views/test_statistics_view.py @@ -0,0 +1,52 @@ +from greedybear.models import GeneralHoneypot, Statistics, ViewType +from tests import CustomTestCase + + +class StatisticsViewTestCase(CustomTestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + Statistics.objects.all().delete() + Statistics.objects.create(source="140.246.171.141", view=ViewType.FEEDS_VIEW.value) + Statistics.objects.create(source="140.246.171.141", view=ViewType.ENRICHMENT_VIEW.value) + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + Statistics.objects.all().delete() + + def test_200_feeds_sources(self): + response = self.client.get("/api/statistics/sources/feeds") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()[0]["Sources"], 1) + + def test_200_feeds_downloads(self): + response = self.client.get("/api/statistics/downloads/feeds") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()[0]["Downloads"], 1) + + def test_200_enrichment_sources(self): + response = self.client.get("/api/statistics/sources/enrichment") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()[0]["Sources"], 1) + + def test_200_enrichment_requests(self): + response = self.client.get("/api/statistics/requests/enrichment") + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()[0]["Requests"], 1) + + def test_200_feed_types(self): + # Count honeypots before adding new one + initial_count = GeneralHoneypot.objects.count() + # add a general honeypot without associated ioc + GeneralHoneypot(name="Tanner", active=True).save() + self.assertEqual(GeneralHoneypot.objects.count(), initial_count + 1) + + response = self.client.get("/api/statistics/feeds_types") + self.assertEqual(response.status_code, 200) + # Expecting 3 because setupTestData creates 3 IOCs (ioc, ioc_2, ioc_domain) associated with Heralding + self.assertEqual(response.json()[0]["Heralding"], 3) + self.assertEqual(response.json()[0]["Ciscoasa"], 2) + self.assertEqual(response.json()[0]["Log4pot"], 3) + self.assertEqual(response.json()[0]["Cowrie"], 3) + self.assertEqual(response.json()[0]["Tanner"], 0) diff --git a/tests/api/views/test_validation_helpers.py b/tests/api/views/test_validation_helpers.py new file mode 100644 index 00000000..2e9f900e --- /dev/null +++ b/tests/api/views/test_validation_helpers.py @@ -0,0 +1,38 @@ +from api.views.utils import is_ip_address, is_sha256hash +from tests import CustomTestCase + + +class ValidationHelpersTestCase(CustomTestCase): + """Test cases for the validation helper functions.""" + + def test_is_ip_address_valid_ipv4(self): + """Test that is_ip_address returns True for valid IPv4 addresses.""" + self.assertTrue(is_ip_address("192.168.1.1")) + self.assertTrue(is_ip_address("10.0.0.1")) + self.assertTrue(is_ip_address("127.0.0.1")) + + def test_is_ip_address_valid_ipv6(self): + """Test that is_ip_address returns True for valid IPv6 addresses.""" + self.assertTrue(is_ip_address("::1")) + self.assertTrue(is_ip_address("2001:db8::1")) + self.assertTrue(is_ip_address("fe80::1ff:fe23:4567:890a")) + + def test_is_ip_address_invalid(self): + """Test that is_ip_address returns False for invalid IP addresses.""" + self.assertFalse(is_ip_address("not-an-ip")) + self.assertFalse(is_ip_address("256.256.256.256")) + self.assertFalse(is_ip_address("192.168.0")) + self.assertFalse(is_ip_address("2001:xyz::1")) + + def test_is_sha256hash_valid(self): + """Test that is_sha256hash returns True for valid SHA-256 hashes.""" + self.assertTrue(is_sha256hash("a" * 64)) + self.assertTrue(is_sha256hash("1234567890abcdef" * 4)) + self.assertTrue(is_sha256hash("A" * 64)) + + def test_is_sha256hash_invalid(self): + """Test that is_sha256hash returns False for invalid SHA-256 hashes.""" + self.assertFalse(is_sha256hash("a" * 63)) # Too short + self.assertFalse(is_sha256hash("a" * 65)) # Too long + self.assertFalse(is_sha256hash("z" * 64)) # Invalid chars + self.assertFalse(is_sha256hash("not-a-hash")) diff --git a/tests/greedybear/cronjobs/test_extraction_pipeline_e2e.py b/tests/greedybear/cronjobs/test_extraction_pipeline_e2e.py index 54104caf..be7a66c3 100644 --- a/tests/greedybear/cronjobs/test_extraction_pipeline_e2e.py +++ b/tests/greedybear/cronjobs/test_extraction_pipeline_e2e.py @@ -49,7 +49,7 @@ def test_cowrie_extracts_scanner_ioc(self, mock_session_repo, mock_scores): result = pipeline.execute() # Verify sensor was extracted - pipeline.sensor_repo.add_sensor.assert_called_with("10.0.0.1") + pipeline.sensor_repo.get_or_create_sensor.assert_called_with("10.0.0.1") # Verify IOC was created self.assertGreaterEqual(result, 0) @@ -120,7 +120,7 @@ def test_unknown_honeypot_uses_generic_strategy(self, mock_scores): result = pipeline.execute() # Sensor should be registered - pipeline.sensor_repo.add_sensor.assert_called_with("10.0.0.5") + pipeline.sensor_repo.get_or_create_sensor.assert_called_with("10.0.0.5") self.assertGreaterEqual(result, 0) diff --git a/tests/greedybear/cronjobs/test_extraction_pipeline_grouping.py b/tests/greedybear/cronjobs/test_extraction_pipeline_grouping.py index c61e6346..4adc62ae 100644 --- a/tests/greedybear/cronjobs/test_extraction_pipeline_grouping.py +++ b/tests/greedybear/cronjobs/test_extraction_pipeline_grouping.py @@ -101,7 +101,7 @@ def test_extracts_sensor_from_hits(self, mock_factory, mock_scores): pipeline.execute() - pipeline.sensor_repo.add_sensor.assert_called_once_with("10.0.0.1") + pipeline.sensor_repo.get_or_create_sensor.assert_called_once_with("10.0.0.1") pipeline.elastic_repo.search.assert_called_once_with(10) @patch("greedybear.cronjobs.extraction.pipeline.UpdateScores") @@ -129,7 +129,7 @@ def test_sensor_not_extracted_for_invalid_hits(self, mock_factory, mock_scores): pipeline.execute() # Sensor should NOT be extracted for invalid hits (missing type) - pipeline.sensor_repo.add_sensor.assert_not_called() + pipeline.sensor_repo.get_or_create_sensor.assert_not_called() class TestHitGrouping(ExtractionPipelineTestCase): diff --git a/tests/test_cowrie_extraction.py b/tests/test_cowrie_extraction.py index b559d726..020dd51e 100644 --- a/tests/test_cowrie_extraction.py +++ b/tests/test_cowrie_extraction.py @@ -332,6 +332,7 @@ def test_deduplicate_command_sequence_existing(self): def test_extract_from_hits_integration(self, mock_iocs_from_hits): """Test the main extract_from_hits coordination.""" mock_ioc = Mock(name="1.2.3.4") + # Return list of IOCs as expected by the new format mock_iocs_from_hits.return_value = [mock_ioc] mock_ioc_record = Mock() diff --git a/tests/test_extraction_strategies.py b/tests/test_extraction_strategies.py index c5b83f99..6aaec4f8 100644 --- a/tests/test_extraction_strategies.py +++ b/tests/test_extraction_strategies.py @@ -85,3 +85,27 @@ def test_logs_correct_honeypot_name(self, mock_iocs_from_hits): call_kwargs = self.strategy.ioc_processor.add_ioc.call_args[1] self.assertEqual(call_kwargs["general_honeypot_name"], "TestHoneypot") + + @patch("greedybear.cronjobs.extraction.strategies.generic.iocs_from_hits") + @patch("greedybear.cronjobs.extraction.strategies.generic.threatfox_submission") + def test_processes_ioc_with_sensors(self, mock_threatfox, mock_iocs_from_hits): + """Test that sensors are passed to add_ioc when present""" + self.mock_ioc_repo.is_enabled.return_value = True + + mock_ioc = self._create_mock_ioc() + mock_sensor1 = Mock() + mock_sensor1.address = "10.0.0.1" + mock_sensor2 = Mock() + mock_sensor2.address = "10.0.0.2" + # Attach sensors to IOC + mock_ioc._sensors_to_add = [mock_sensor1, mock_sensor2] + mock_iocs_from_hits.return_value = [mock_ioc] + + self.strategy.ioc_processor.add_ioc = Mock(return_value=mock_ioc) + + hits = [{"src_ip": "1.2.3.4", "dest_port": 80, "@timestamp": "2025-01-01T00:00:00"}] + + self.strategy.extract_from_hits(hits) + + # Should call add_ioc once with IOC object (sensors are attached to it) + self.strategy.ioc_processor.add_ioc.assert_called_once_with(mock_ioc, attack_type=SCANNER, general_honeypot_name="TestHoneypot") diff --git a/tests/test_extraction_utils.py b/tests/test_extraction_utils.py index 558e76d3..abdeb40e 100644 --- a/tests/test_extraction_utils.py +++ b/tests/test_extraction_utils.py @@ -325,6 +325,7 @@ def _create_hit( hit_type="Cowrie", ip_rep="", asn=None, + sensor=None, ): hit = { "src_ip": src_ip, @@ -335,14 +336,17 @@ def _create_hit( } if asn: hit["geoip"] = {"asn": asn} + if sensor: + hit["_sensor"] = sensor return hit def test_creates_ioc_from_single_hit(self): hits = [self._create_hit(src_ip="8.8.8.8", dest_port=22)] iocs = iocs_from_hits(hits) self.assertEqual(len(iocs), 1) - self.assertEqual(iocs[0].name, "8.8.8.8") - self.assertEqual(iocs[0].type, IP) + ioc = iocs[0] + self.assertEqual(ioc.name, "8.8.8.8") + self.assertEqual(ioc.type, IP) def test_groups_hits_by_ip(self): hits = [ @@ -352,8 +356,9 @@ def test_groups_hits_by_ip(self): ] iocs = iocs_from_hits(hits) self.assertEqual(len(iocs), 1) - self.assertEqual(iocs[0].attack_count, 1) - self.assertEqual(iocs[0].interaction_count, 3) + ioc = iocs[0] + self.assertEqual(ioc.attack_count, 1) + self.assertEqual(ioc.interaction_count, 3) def test_creates_separate_iocs_for_different_ips(self): hits = [ @@ -372,7 +377,8 @@ def test_aggregates_destination_ports(self): self._create_hit(src_ip="8.8.8.8", dest_port=443), ] iocs = iocs_from_hits(hits) - self.assertEqual(iocs[0].destination_ports, [22, 80, 443]) + ioc = iocs[0] + self.assertEqual(ioc.destination_ports, [22, 80, 443]) def test_deduplicates_ports(self): hits = [ @@ -381,24 +387,28 @@ def test_deduplicates_ports(self): self._create_hit(src_ip="8.8.8.8", dest_port=22), ] iocs = iocs_from_hits(hits) - self.assertEqual(iocs[0].destination_ports, [22]) + ioc = iocs[0] + self.assertEqual(ioc.destination_ports, [22]) def test_handles_missing_dest_port(self): hits = [ {"src_ip": "8.8.8.8", "@timestamp": "2025-01-01T12:00:00.000Z"}, ] iocs = iocs_from_hits(hits) - self.assertEqual(iocs[0].destination_ports, []) + ioc = iocs[0] + self.assertEqual(ioc.destination_ports, []) def test_extracts_asn_from_geoip(self): hits = [self._create_hit(src_ip="8.8.8.8", asn=15169)] iocs = iocs_from_hits(hits) - self.assertEqual(iocs[0].asn, 15169) + ioc = iocs[0] + self.assertEqual(ioc.asn, 15169) def test_handles_missing_geoip(self): hits = [{"src_ip": "8.8.8.8", "@timestamp": "2025-01-01T12:00:00.000Z"}] iocs = iocs_from_hits(hits) - self.assertIsNone(iocs[0].asn) + ioc = iocs[0] + self.assertIsNone(ioc.asn) def test_extracts_timestamps(self): hits = [ @@ -407,8 +417,9 @@ def test_extracts_timestamps(self): self._create_hit(src_ip="8.8.8.8", timestamp="2025-01-01T11:00:00.000Z"), ] iocs = iocs_from_hits(hits) - self.assertEqual(iocs[0].first_seen, datetime.fromisoformat("2025-01-01T10:00:00.000Z")) - self.assertEqual(iocs[0].last_seen, datetime.fromisoformat("2025-01-01T12:00:00.000Z")) + ioc = iocs[0] + self.assertEqual(ioc.first_seen, datetime.fromisoformat("2025-01-01T10:00:00.000Z")) + self.assertEqual(ioc.last_seen, datetime.fromisoformat("2025-01-01T12:00:00.000Z")) def test_filters_loopback_addresses(self): hits = [ @@ -417,7 +428,8 @@ def test_filters_loopback_addresses(self): ] iocs = iocs_from_hits(hits) self.assertEqual(len(iocs), 1) - self.assertEqual(iocs[0].name, "8.8.8.8") + ioc = iocs[0] + self.assertEqual(ioc.name, "8.8.8.8") def test_filters_private_addresses(self): hits = [ @@ -428,7 +440,8 @@ def test_filters_private_addresses(self): ] iocs = iocs_from_hits(hits) self.assertEqual(len(iocs), 1) - self.assertEqual(iocs[0].name, "8.8.8.8") + ioc = iocs[0] + self.assertEqual(ioc.name, "8.8.8.8") def test_filters_multicast_addresses(self): hits = [ @@ -437,7 +450,8 @@ def test_filters_multicast_addresses(self): ] iocs = iocs_from_hits(hits) self.assertEqual(len(iocs), 1) - self.assertEqual(iocs[0].name, "8.8.8.8") + ioc = iocs[0] + self.assertEqual(ioc.name, "8.8.8.8") def test_filters_link_local_addresses(self): hits = [ @@ -446,7 +460,8 @@ def test_filters_link_local_addresses(self): ] iocs = iocs_from_hits(hits) self.assertEqual(len(iocs), 1) - self.assertEqual(iocs[0].name, "8.8.8.8") + ioc = iocs[0] + self.assertEqual(ioc.name, "8.8.8.8") def test_filters_reserved_addresses(self): hits = [ @@ -455,7 +470,8 @@ def test_filters_reserved_addresses(self): ] iocs = iocs_from_hits(hits) self.assertEqual(len(iocs), 1) - self.assertEqual(iocs[0].name, "8.8.8.8") + ioc = iocs[0] + self.assertEqual(ioc.name, "8.8.8.8") def test_heralding_counts_login_attempts(self): hits = [ @@ -464,7 +480,8 @@ def test_heralding_counts_login_attempts(self): self._create_hit(src_ip="8.8.8.8", hit_type="Heralding"), ] iocs = iocs_from_hits(hits) - self.assertEqual(iocs[0].login_attempts, 3) + ioc = iocs[0] + self.assertEqual(ioc.login_attempts, 3) def test_non_heralding_no_login_attempts(self): hits = [ @@ -472,13 +489,15 @@ def test_non_heralding_no_login_attempts(self): self._create_hit(src_ip="8.8.8.8", hit_type="Cowrie"), ] iocs = iocs_from_hits(hits) - self.assertEqual(iocs[0].login_attempts, 0) + ioc = iocs[0] + self.assertEqual(ioc.login_attempts, 0) def test_corrects_ip_reputation(self): MassScanner.objects.create(ip_address="8.8.8.8") hits = [self._create_hit(src_ip="8.8.8.8", ip_rep="known attacker")] iocs = iocs_from_hits(hits) - self.assertEqual(iocs[0].ip_reputation, "mass scanner") + ioc = iocs[0] + self.assertEqual(ioc.ip_reputation, "mass scanner") def test_empty_hits_returns_empty_list(self): iocs = iocs_from_hits([]) @@ -493,9 +512,10 @@ def test_firehol_enrichment_exact_ip_match(self): iocs = iocs_from_hits(hits) self.assertEqual(len(iocs), 1) - self.assertIn("blocklist_de", iocs[0].firehol_categories) - self.assertIn("greensnow", iocs[0].firehol_categories) - self.assertEqual(len(iocs[0].firehol_categories), 2) + ioc = iocs[0] + self.assertIn("blocklist_de", ioc.firehol_categories) + self.assertIn("greensnow", ioc.firehol_categories) + self.assertEqual(len(ioc.firehol_categories), 2) def test_firehol_enrichment_network_range_match(self): """Test that IOCs get FireHol categories when IP is within a CIDR range (.netset files)""" @@ -505,7 +525,8 @@ def test_firehol_enrichment_network_range_match(self): iocs = iocs_from_hits(hits) self.assertEqual(len(iocs), 1) - self.assertIn("dshield", iocs[0].firehol_categories) + ioc = iocs[0] + self.assertIn("dshield", ioc.firehol_categories) def test_firehol_enrichment_no_match(self): """Test that IOCs have empty FireHol categories when there's no match""" @@ -516,7 +537,8 @@ def test_firehol_enrichment_no_match(self): iocs = iocs_from_hits(hits) self.assertEqual(len(iocs), 1) - self.assertEqual(iocs[0].firehol_categories, []) + ioc = iocs[0] + self.assertEqual(ioc.firehol_categories, []) def test_firehol_enrichment_mixed_match(self): """Test FireHol enrichment with both exact match and network range match""" @@ -527,8 +549,9 @@ def test_firehol_enrichment_mixed_match(self): iocs = iocs_from_hits(hits) self.assertEqual(len(iocs), 1) - self.assertIn("blocklist_de", iocs[0].firehol_categories) - self.assertIn("dshield", iocs[0].firehol_categories) + ioc = iocs[0] + self.assertIn("blocklist_de", ioc.firehol_categories) + self.assertIn("dshield", ioc.firehol_categories) def test_firehol_enrichment_deduplicates_sources(self): """Test that duplicate sources are not added""" @@ -539,8 +562,40 @@ def test_firehol_enrichment_deduplicates_sources(self): iocs = iocs_from_hits(hits) self.assertEqual(len(iocs), 1) + ioc = iocs[0] # Should only have one instance of blocklist_de - self.assertEqual(iocs[0].firehol_categories.count("blocklist_de"), 1) + self.assertEqual(ioc.firehol_categories.count("blocklist_de"), 1) + + def test_collects_sensors_from_hits(self): + """Test that sensors are collected from hits and returned""" + from greedybear.models import Sensor + + sensor1 = Sensor.objects.create(address="10.0.0.1") + sensor2 = Sensor.objects.create(address="10.0.0.2") + + hits = [ + self._create_hit(src_ip="8.8.8.8", sensor=sensor1), + self._create_hit(src_ip="8.8.8.8", sensor=sensor2), + self._create_hit(src_ip="8.8.8.8", sensor=sensor1), # Duplicate - should be deduplicated + ] + iocs = iocs_from_hits(hits) + + self.assertEqual(len(iocs), 1) + ioc = iocs[0] + sensors = ioc._sensors_to_add + self.assertEqual(len(sensors), 2) + sensor_addresses = {s.address for s in sensors} + self.assertEqual(sensor_addresses, {"10.0.0.1", "10.0.0.2"}) + + def test_returns_empty_sensors_when_no_sensor_in_hits(self): + """Test that empty sensor list is returned when hits have no sensor""" + hits = [self._create_hit(src_ip="8.8.8.8")] + iocs = iocs_from_hits(hits) + + self.assertEqual(len(iocs), 1) + ioc = iocs[0] + sensor = getattr(ioc, "_sensors_to_add", []) + self.assertEqual(sensor, []) class ThreatfoxSubmissionTestCase(ExtractionTestCase): diff --git a/tests/test_ioc_processor.py b/tests/test_ioc_processor.py index 5b68153e..6941c9c2 100644 --- a/tests/test_ioc_processor.py +++ b/tests/test_ioc_processor.py @@ -1,5 +1,5 @@ from datetime import date, datetime -from unittest.mock import patch +from unittest.mock import Mock, patch from greedybear.consts import PAYLOAD_REQUEST, SCANNER from greedybear.cronjobs.extraction.ioc_processor import IocProcessor @@ -14,7 +14,7 @@ def setUp(self): self.processor = IocProcessor(self.mock_ioc_repo, self.mock_sensor_repo) def test_filters_sensor_ips(self): - self.mock_sensor_repo.sensors = {"192.168.1.1"} + self.mock_sensor_repo.cache = {"192.168.1.1": Mock()} ioc = self._create_mock_ioc(name="192.168.1.1") result = self.processor.add_ioc(ioc, attack_type=SCANNER) @@ -25,7 +25,6 @@ def test_filters_sensor_ips(self): @patch("greedybear.cronjobs.extraction.ioc_processor.is_whatsmyip_domain") def test_filters_whatsmyip_domains(self, mock_whatsmyip): mock_whatsmyip.return_value = True - self.mock_sensor_repo.sensors = set() ioc = self._create_mock_ioc(name="some.domain.com", ioc_type=IocType.DOMAIN) result = self.processor.add_ioc(ioc, attack_type=SCANNER) @@ -35,7 +34,7 @@ def test_filters_whatsmyip_domains(self, mock_whatsmyip): self.mock_ioc_repo.save.assert_not_called() def test_creates_new_ioc_when_not_exists(self): - self.mock_sensor_repo.sensors = set() + self.mock_sensor_repo.cache = {} self.mock_ioc_repo.get_ioc_by_name.return_value = None new_ioc = self._create_mock_ioc() self.mock_ioc_repo.save.return_value = new_ioc @@ -47,7 +46,7 @@ def test_creates_new_ioc_when_not_exists(self): self.assertIsNotNone(result) def test_updates_existing_ioc_when_exists(self): - self.mock_sensor_repo.sensors = set() + self.mock_sensor_repo.cache = {} existing_ioc = self._create_mock_ioc(attack_count=5) self.mock_ioc_repo.get_ioc_by_name.return_value = existing_ioc new_ioc = self._create_mock_ioc(attack_count=1) @@ -59,7 +58,7 @@ def test_updates_existing_ioc_when_exists(self): self.assertEqual(result.attack_count, 6) def test_sets_scanner_flag_for_scanner_attack_type(self): - self.mock_sensor_repo.sensors = set() + self.mock_sensor_repo.cache = {} self.mock_ioc_repo.get_ioc_by_name.return_value = None ioc = self._create_mock_ioc() self.mock_ioc_repo.save.return_value = ioc @@ -70,7 +69,7 @@ def test_sets_scanner_flag_for_scanner_attack_type(self): self.assertFalse(result.payload_request) def test_sets_payload_request_flag_for_payload_attack_type(self): - self.mock_sensor_repo.sensors = set() + self.mock_sensor_repo.cache = {} self.mock_ioc_repo.get_ioc_by_name.return_value = None ioc = self._create_mock_ioc() self.mock_ioc_repo.save.return_value = ioc @@ -81,7 +80,7 @@ def test_sets_payload_request_flag_for_payload_attack_type(self): self.assertTrue(result.payload_request) def test_adds_general_honeypot_when_provided(self): - self.mock_sensor_repo.sensors = set() + self.mock_sensor_repo.cache = {} self.mock_ioc_repo.get_ioc_by_name.return_value = None ioc = self._create_mock_ioc() self.mock_ioc_repo.save.return_value = ioc @@ -92,7 +91,7 @@ def test_adds_general_honeypot_when_provided(self): self.mock_ioc_repo.add_honeypot_to_ioc.assert_called_once_with("TestHoneypot", ioc) def test_skips_general_honeypot_when_not_provided(self): - self.mock_sensor_repo.sensors = set() + self.mock_sensor_repo.cache = {} self.mock_ioc_repo.get_ioc_by_name.return_value = None ioc = self._create_mock_ioc() self.mock_ioc_repo.save.return_value = ioc @@ -101,8 +100,23 @@ def test_skips_general_honeypot_when_not_provided(self): self.mock_ioc_repo.add_honeypot_to_ioc.assert_not_called() + def test_adds_sensors_from_attribute(self): + self.mock_sensor_repo.cache = {} + self.mock_ioc_repo.get_ioc_by_name.return_value = None + ioc = self._create_mock_ioc() + sensor1 = Mock() + sensor2 = Mock() + ioc._sensors_to_add = [sensor1, sensor2] + + self.mock_ioc_repo.save.return_value = ioc + + self.processor.add_ioc(ioc, attack_type=SCANNER) + + ioc.sensors.add.assert_any_call(sensor1) + ioc.sensors.add.assert_any_call(sensor2) + def test_updates_days_seen_on_add(self): - self.mock_sensor_repo.sensors = set() + self.mock_sensor_repo.cache = {} self.mock_ioc_repo.get_ioc_by_name.return_value = None ioc = self._create_mock_ioc(days_seen=[], last_seen=datetime(2025, 1, 1, 12, 0, 0)) self.mock_ioc_repo.save.return_value = ioc @@ -113,7 +127,7 @@ def test_updates_days_seen_on_add(self): self.assertEqual(result.number_of_days_seen, 1) def test_full_create_flow(self): - self.mock_sensor_repo.sensors = set() + self.mock_sensor_repo.cache = {} self.mock_ioc_repo.get_ioc_by_name.return_value = None ioc = self._create_mock_ioc( @@ -133,7 +147,7 @@ def test_full_create_flow(self): self.assertEqual(len(result.days_seen), 1) def test_full_update_flow(self): - self.mock_sensor_repo.sensors = set() + self.mock_sensor_repo.cache = {} existing = self._create_mock_ioc( attack_count=5, @@ -166,7 +180,7 @@ def test_full_update_flow(self): @patch("greedybear.cronjobs.extraction.ioc_processor.is_whatsmyip_domain") def test_only_checks_whatsmyip_for_domains(self, mock_whatsmyip): - self.mock_sensor_repo.sensors = set() + self.mock_sensor_repo.cache = {} self.mock_ioc_repo.get_ioc_by_name.return_value = None ioc = self._create_mock_ioc(name="1.2.3.4", ioc_type=IocType.IP) self.mock_ioc_repo.save.return_value = ioc diff --git a/tests/test_migrations.py b/tests/test_migrations.py index 04898b0b..610317a5 100644 --- a/tests/test_migrations.py +++ b/tests/test_migrations.py @@ -74,3 +74,48 @@ def test_boolean_flags_are_migrated_to_m2m(self): ioc_new.objects.get(id=ioc_none.id).general_honeypot.count(), 0, ) + + +@tag("migration") +class TestRemoveUnusedLog4pot(MigrationTestCase): + """Tests that Log4pot is removed only when it has no associated IOCs. + + Fixes issue #773: Log4pot is active despite having no data. + """ + + migrate_from = "0033_disable_additional_honeypots" + migrate_to = "0034_remove_unused_log4pot" + + def test_log4pot_deleted_if_unused(self): + """Log4pot should be deleted if it has no associated IOCs.""" + GeneralHoneypot = self.old_state.apps.get_model(self.app_name, "GeneralHoneypot") + + # Ensure Log4pot exists (created by migration 0030) + GeneralHoneypot.objects.get_or_create(name="Log4pot", defaults={"active": True}) + + new_state = self.apply_tested_migration() + hp_new = new_state.apps.get_model(self.app_name, "GeneralHoneypot") + + self.assertFalse( + hp_new.objects.filter(name="Log4pot").exists(), + "Log4pot with no IOCs should be deleted", + ) + + def test_log4pot_kept_if_has_iocs(self): + """Log4pot should NOT be deleted if it has associated IOCs.""" + GeneralHoneypot = self.old_state.apps.get_model(self.app_name, "GeneralHoneypot") + IOC = self.old_state.apps.get_model(self.app_name, "IOC") + + log4pot_hp, _ = GeneralHoneypot.objects.get_or_create(name="Log4pot", defaults={"active": True}) + + # Create an IOC and link it to Log4pot + ioc = IOC.objects.create() + ioc.general_honeypot.add(log4pot_hp) + + new_state = self.apply_tested_migration() + hp_new = new_state.apps.get_model(self.app_name, "GeneralHoneypot") + + self.assertTrue( + hp_new.objects.filter(name="Log4pot").exists(), + "Log4pot with IOCs should NOT be deleted", + ) diff --git a/tests/test_sensor_repository.py b/tests/test_sensor_repository.py index 1220ad4a..09c9f626 100644 --- a/tests/test_sensor_repository.py +++ b/tests/test_sensor_repository.py @@ -8,34 +8,28 @@ class TestSensorRepository(CustomTestCase): def setUp(self): self.repo = SensorRepository() - def test_sensors_property_returns_cached_sensors(self): - self.repo.add_sensor("192.168.1.1") - self.repo.add_sensor("192.168.1.2") - result = self.repo.sensors - self.assertEqual(len(result), 2) - self.assertIn("192.168.1.1", result) - self.assertIn("192.168.1.2", result) - - def test_add_sensor_creates_new_sensor(self): - result = self.repo.add_sensor("192.168.1.3") - self.assertTrue(result) + def test_get_or_create_sensor_creates_new_sensor(self): + result = self.repo.get_or_create_sensor("192.168.1.3") + self.assertIsNotNone(result) + self.assertIsInstance(result, Sensor) + self.assertEqual(result.address, "192.168.1.3") self.assertTrue(Sensor.objects.filter(address="192.168.1.3").exists()) self.assertIn("192.168.1.3", self.repo.cache) - def test_add_sensor_returns_false_for_existing_sensor(self): - self.repo.add_sensor("192.168.1.1") - result = self.repo.add_sensor("192.168.1.1") - self.assertFalse(result) + def test_get_or_create_sensor_returns_existing_sensor(self): + first_result = self.repo.get_or_create_sensor("192.168.1.1") + second_result = self.repo.get_or_create_sensor("192.168.1.1") + self.assertEqual(first_result.pk, second_result.pk) self.assertEqual(Sensor.objects.filter(address="192.168.1.1").count(), 1) - def test_add_sensor_rejects_non_ip(self): - result = self.repo.add_sensor("not-an-ip") - self.assertFalse(result) + def test_get_or_create_sensor_rejects_non_ip(self): + result = self.repo.get_or_create_sensor("not-an-ip") + self.assertIsNone(result) self.assertFalse(Sensor.objects.filter(address="not-an-ip").exists()) - def test_add_sensor_rejects_domain(self): - result = self.repo.add_sensor("example.com") - self.assertFalse(result) + def test_get_or_create_sensor_rejects_domain(self): + result = self.repo.get_or_create_sensor("example.com") + self.assertIsNone(result) self.assertFalse(Sensor.objects.filter(address="example.com").exists()) def test_cache_populated_on_init(self): @@ -46,13 +40,21 @@ def test_cache_populated_on_init(self): self.assertIn("192.168.1.1", repo.cache) self.assertIn("192.168.1.2", repo.cache) - def test_add_sensor_updates_cache(self): + def test_cache_stores_sensor_objects(self): + sensor = Sensor.objects.create(address="192.168.1.1") + repo = SensorRepository() + cached_sensor = repo.cache.get("192.168.1.1") + self.assertIsInstance(cached_sensor, Sensor) + self.assertEqual(cached_sensor.pk, sensor.pk) + + def test_get_or_create_sensor_updates_cache(self): initial_cache_size = len(self.repo.cache) - self.repo.add_sensor("192.168.1.1") + self.repo.get_or_create_sensor("192.168.1.1") self.assertEqual(len(self.repo.cache), initial_cache_size + 1) - def test_add_sensor_accepts_valid_ipv4(self): + def test_get_or_create_sensor_accepts_valid_ipv4(self): test_ips = ["1.2.3.4", "192.168.1.1", "10.0.0.1", "8.8.8.8"] for ip in test_ips: - result = self.repo.add_sensor(ip) - self.assertTrue(result) + result = self.repo.get_or_create_sensor(ip) + self.assertIsNotNone(result) + self.assertIsInstance(result, Sensor) diff --git a/tests/test_views.py b/tests/test_views.py deleted file mode 100644 index c09f01ea..00000000 --- a/tests/test_views.py +++ /dev/null @@ -1,867 +0,0 @@ -from django.conf import settings -from django.test import override_settings -from django.utils import timezone -from rest_framework.test import APIClient - -from api.views.utils import is_ip_address, is_sha256hash -from greedybear.models import IOC, GeneralHoneypot, Statistics, ViewType - -from . import CustomTestCase - - -class EnrichmentViewTestCase(CustomTestCase): - def setUp(self): - # setup client - self.client = APIClient() - self.client.force_authenticate(user=self.superuser) - - def test_for_vaild_unregistered_ip(self): - """Check for a valid IP that is unavaliable in DB""" - response = self.client.get("/api/enrichment?query=192.168.0.1") - self.assertEqual(response.status_code, 200) - self.assertEqual(response.json()["found"], False) - - def test_for_invaild_unregistered_ip(self): - """Check for a IP that Fails Regex Checks and is unavaliable in DB""" - response = self.client.get("/api/enrichment?query=30.168.1.255.1") - self.assertEqual(response.status_code, 400) - - def test_for_vaild_registered_ip(self): - """Check for a valid IP that is avaliable in DB""" - response = self.client.get("/api/enrichment?query=140.246.171.141") - self.assertEqual(response.status_code, 200) - self.assertEqual(response.json()["found"], True) - self.assertEqual(response.json()["ioc"]["name"], self.ioc.name) - self.assertEqual(response.json()["ioc"]["type"], self.ioc.type) - self.assertEqual( - response.json()["ioc"]["first_seen"], - self.ioc.first_seen.isoformat(sep="T", timespec="microseconds"), - ) - self.assertEqual( - response.json()["ioc"]["last_seen"], - self.ioc.last_seen.isoformat(sep="T", timespec="microseconds"), - ) - self.assertEqual(response.json()["ioc"]["number_of_days_seen"], self.ioc.number_of_days_seen) - self.assertEqual(response.json()["ioc"]["attack_count"], self.ioc.attack_count) - # Honeypots are now via M2M relationship (serialized as list of strings) - honeypot_names = response.json()["ioc"]["general_honeypot"] - self.assertIn(self.heralding.name, honeypot_names) - self.assertIn(self.ciscoasa.name, honeypot_names) - self.assertIn(self.cowrie_hp.name, honeypot_names) - self.assertIn(self.log4pot_hp.name, honeypot_names) - self.assertEqual(response.json()["ioc"]["scanner"], self.ioc.scanner) - self.assertEqual(response.json()["ioc"]["payload_request"], self.ioc.payload_request) - self.assertEqual( - response.json()["ioc"]["recurrence_probability"], - self.ioc.recurrence_probability, - ) - self.assertEqual( - response.json()["ioc"]["expected_interactions"], - self.ioc.expected_interactions, - ) - - def test_for_invalid_authentication(self): - """Check for a invalid authentication""" - self.client.logout() - response = self.client.get("/api/enrichment?query=140.246.171.141") - self.assertEqual(response.status_code, 401) - - -class FeedsViewTestCase(CustomTestCase): - def test_200_log4pot_feeds(self): - response = self.client.get("/api/feeds/log4pot/all/recent.json") - self.assertEqual(response.status_code, 200) - if settings.FEEDS_LICENSE: - self.assertEqual(response.json()["license"], settings.FEEDS_LICENSE) - else: - self.assertNotIn("license", response.json()) - - iocs = response.json()["iocs"] - target_ioc = next((i for i in iocs if i["value"] == self.ioc.name), None) - self.assertIsNotNone(target_ioc) - - # feed_type now derived from general_honeypot M2M - self.assertIn("log4pot", target_ioc["feed_type"]) - self.assertIn("cowrie", target_ioc["feed_type"]) - self.assertIn("heralding", target_ioc["feed_type"]) - self.assertIn("ciscoasa", target_ioc["feed_type"]) - self.assertEqual(target_ioc["attack_count"], 1) - self.assertEqual(target_ioc["scanner"], True) - self.assertEqual(target_ioc["payload_request"], True) - self.assertEqual(target_ioc["recurrence_probability"], self.ioc.recurrence_probability) - self.assertEqual(target_ioc["expected_interactions"], self.ioc.expected_interactions) - - @override_settings(FEEDS_LICENSE="https://example.com/license") - def test_200_all_feeds_with_license(self): - """Test feeds endpoint when FEEDS_LICENSE is populated""" - response = self.client.get("/api/feeds/all/all/recent.json") - self.assertEqual(response.status_code, 200) - self.assertIn("license", response.json()) - self.assertEqual(response.json()["license"], "https://example.com/license") - - @override_settings(FEEDS_LICENSE="") - def test_200_all_feeds_without_license(self): - """Test feeds endpoint when FEEDS_LICENSE is empty""" - response = self.client.get("/api/feeds/all/all/recent.json") - self.assertEqual(response.status_code, 200) - self.assertNotIn("license", response.json()) - - def test_200_general_feeds(self): - response = self.client.get("/api/feeds/heralding/all/recent.json") - self.assertEqual(response.status_code, 200) - if settings.FEEDS_LICENSE: - self.assertEqual(response.json()["license"], settings.FEEDS_LICENSE) - else: - self.assertNotIn("license", response.json()) - - iocs = response.json()["iocs"] - target_ioc = next((i for i in iocs if i["value"] == self.ioc.name), None) - self.assertIsNotNone(target_ioc) - - self.assertEqual(set(target_ioc["feed_type"]), {"log4pot", "cowrie", "heralding", "ciscoasa"}) - self.assertEqual(target_ioc["attack_count"], 1) - self.assertEqual(target_ioc["scanner"], True) - self.assertEqual(target_ioc["payload_request"], True) - self.assertEqual(target_ioc["recurrence_probability"], self.ioc.recurrence_probability) - self.assertEqual(target_ioc["expected_interactions"], self.ioc.expected_interactions) - - def test_200_feeds_scanner_inclusion(self): - response = self.client.get("/api/feeds/heralding/all/recent.json?include_mass_scanners") - self.assertEqual(response.status_code, 200) - if settings.FEEDS_LICENSE: - self.assertEqual(response.json()["license"], settings.FEEDS_LICENSE) - else: - self.assertNotIn("license", response.json()) - # Expecting 3 because setupTestData creates 3 IOCs (ioc, ioc_2, ioc_domain) associated with Heralding - self.assertEqual(len(response.json()["iocs"]), 3) - - def test_400_feeds(self): - response = self.client.get("/api/feeds/test/all/recent.json") - self.assertEqual(response.status_code, 400) - - def test_200_feeds_pagination(self): - response = self.client.get("/api/feeds/?page_size=10&page=1&feed_type=all&attack_type=all&age=recent") - self.assertEqual(response.status_code, 200) - self.assertEqual(response.json()["count"], 2) - self.assertEqual(response.json()["total_pages"], 1) - - def test_200_feeds_pagination_inclusion_mass(self): - response = self.client.get("/api/feeds/?page_size=10&page=1&feed_type=all&attack_type=all&age=recent&include_mass_scanners") - self.assertEqual(response.status_code, 200) - self.assertEqual(response.json()["count"], 3) - - def test_200_feeds_pagination_inclusion_tor(self): - response = self.client.get("/api/feeds/?page_size=10&page=1&feed_type=all&attack_type=all&age=recent&include_tor_exit_nodes") - self.assertEqual(response.status_code, 200) - self.assertEqual(response.json()["count"], 3) - - def test_200_feeds_pagination_inclusion_mass_and_tor(self): - response = self.client.get("/api/feeds/?page_size=10&page=1&feed_type=all&attack_type=all&age=recent&include_mass_scanners&include_tor_exit_nodes") - self.assertEqual(response.status_code, 200) - self.assertEqual(response.json()["count"], 4) - - def test_200_feeds_filter_ip_only(self): - response = self.client.get("/api/feeds/all/all/recent.json?ioc_type=ip") - self.assertEqual(response.status_code, 200) - # Should only return IP addresses, not domains - for ioc in response.json()["iocs"]: - # Verify all returned values are IPs (contain dots and numbers pattern) - self.assertRegex(ioc["value"], r"^\d+\.\d+\.\d+\.\d+$") - - def test_200_feeds_filter_domain_only(self): - response = self.client.get("/api/feeds/all/all/recent.json?ioc_type=domain") - self.assertEqual(response.status_code, 200) - # Should only return domains, not IPs - self.assertGreater(len(response.json()["iocs"]), 0) - for ioc in response.json()["iocs"]: - # Verify all returned values are domains (contain alphabetic characters) - self.assertRegex(ioc["value"], r"[a-zA-Z]") - - def test_200_feeds_pagination_filter_ip(self): - response = self.client.get( - "/api/feeds/?page_size=10&page=1&feed_type=all&attack_type=all&age=recent&ioc_type=ip&include_mass_scanners&include_tor_exit_nodes" - ) - self.assertEqual(response.status_code, 200) - # Should return only IPs (3 in test data) - self.assertEqual(response.json()["count"], 3) - - def test_200_feeds_pagination_filter_domain(self): - response = self.client.get("/api/feeds/?page_size=10&page=1&feed_type=all&attack_type=all&age=recent&ioc_type=domain") - self.assertEqual(response.status_code, 200) - # Should return only domains (1 in test data) - self.assertEqual(response.json()["count"], 1) - - def test_400_feeds_pagination(self): - response = self.client.get("/api/feeds/?page_size=10&page=1&feed_type=all&attack_type=test&age=recent") - self.assertEqual(response.status_code, 400) - - -class FeedsAdvancedViewTestCase(CustomTestCase): - def setUp(self): - self.client = APIClient() - self.client.force_authenticate(user=self.superuser) - - def test_200_all_feeds(self): - response = self.client.get("/api/feeds/advanced/") - self.assertEqual(response.status_code, 200) - if settings.FEEDS_LICENSE: - self.assertEqual(response.json()["license"], settings.FEEDS_LICENSE) - else: - self.assertNotIn("license", response.json()) - - iocs = response.json()["iocs"] - target_ioc = next((i for i in iocs if i["value"] == self.ioc.name), None) - self.assertIsNotNone(target_ioc) - - self.assertEqual(set(target_ioc["feed_type"]), {"log4pot", "cowrie", "heralding", "ciscoasa"}) - self.assertEqual(target_ioc["attack_count"], 1) - self.assertEqual(target_ioc["scanner"], True) - self.assertEqual(target_ioc["payload_request"], True) - self.assertEqual(target_ioc["recurrence_probability"], self.ioc.recurrence_probability) - self.assertEqual(target_ioc["expected_interactions"], self.ioc.expected_interactions) - - def test_200_general_feeds(self): - response = self.client.get("/api/feeds/advanced/?feed_type=heralding") - self.assertEqual(response.status_code, 200) - if settings.FEEDS_LICENSE: - self.assertEqual(response.json()["license"], settings.FEEDS_LICENSE) - else: - self.assertNotIn("license", response.json()) - - iocs = response.json()["iocs"] - target_ioc = next((i for i in iocs if i["value"] == self.ioc.name), None) - self.assertIsNotNone(target_ioc) - - self.assertEqual(set(target_ioc["feed_type"]), {"log4pot", "cowrie", "heralding", "ciscoasa"}) - self.assertEqual(target_ioc["attack_count"], 1) - self.assertEqual(target_ioc["scanner"], True) - self.assertEqual(target_ioc["payload_request"], True) - self.assertEqual(target_ioc["recurrence_probability"], self.ioc.recurrence_probability) - self.assertEqual(target_ioc["expected_interactions"], self.ioc.expected_interactions) - - def test_400_feeds(self): - response = self.client.get("/api/feeds/advanced/?attack_type=test") - self.assertEqual(response.status_code, 400) - - def test_200_feeds_pagination(self): - response = self.client.get("/api/feeds/advanced/?paginate=true&page_size=10&page=1") - self.assertEqual(response.status_code, 200) - self.assertEqual(response.json()["count"], 4) - self.assertEqual(response.json()["total_pages"], 1) - - def test_200_feeds_pagination_include(self): - response = self.client.get("/api/feeds/advanced/?paginate=true&page_size=10&page=1&include_reputation=mass%20scanner") - self.assertEqual(response.status_code, 200) - self.assertEqual(response.json()["count"], 1) - self.assertEqual(response.json()["total_pages"], 1) - - def test_200_feeds_pagination_exclude_mass(self): - response = self.client.get("/api/feeds/advanced/?paginate=true&page_size=10&page=1&exclude_reputation=mass%20scanner") - self.assertEqual(response.status_code, 200) - self.assertEqual(response.json()["count"], 3) - self.assertEqual(response.json()["total_pages"], 1) - - def test_200_feeds_pagination_exclude_tor(self): - response = self.client.get("/api/feeds/advanced/?paginate=true&page_size=10&page=1&exclude_reputation=tor%20exit%20node") - self.assertEqual(response.status_code, 200) - self.assertEqual(response.json()["count"], 3) - self.assertEqual(response.json()["total_pages"], 1) - - def test_400_feeds_pagination(self): - response = self.client.get("/api/feeds/advanced/?paginate=true&page_size=10&page=1&attack_type=test") - self.assertEqual(response.status_code, 400) - - -class FeedsASNViewTestCase(CustomTestCase): - """Tests for ASN aggregated feeds API""" - - @classmethod - def setUpClass(cls): - super().setUpClass() - IOC.objects.all().delete() - cls.testpot1, _ = GeneralHoneypot.objects.get_or_create(name="testpot1", active=True) - cls.testpot2, _ = GeneralHoneypot.objects.get_or_create(name="testpot2", active=True) - - cls.high_asn = "13335" - cls.low_asn = "16276" - - cls.ioc_high1 = IOC.objects.create( - name="high1.example.com", - type="ip", - asn=cls.high_asn, - attack_count=15, - interaction_count=30, - login_attempts=5, - first_seen=timezone.now() - timezone.timedelta(days=10), - recurrence_probability=0.8, - expected_interactions=20.0, - ) - cls.ioc_high1.general_honeypot.add(cls.testpot1, cls.testpot2) - cls.ioc_high1.save() - - cls.ioc_high2 = IOC.objects.create( - name="high2.example.com", - type="ip", - asn=cls.high_asn, - attack_count=5, - interaction_count=10, - login_attempts=2, - first_seen=timezone.now() - timezone.timedelta(days=5), - recurrence_probability=0.3, - expected_interactions=8.0, - ) - cls.ioc_high2.general_honeypot.add(cls.testpot1, cls.testpot2) - cls.ioc_high2.save() - - cls.ioc_low = IOC.objects.create( - name="low.example.com", - type="ip", - asn=cls.low_asn, - attack_count=2, - interaction_count=5, - login_attempts=1, - first_seen=timezone.now(), - recurrence_probability=0.1, - expected_interactions=3.0, - ) - cls.ioc_low.general_honeypot.add(cls.testpot1, cls.testpot2) - cls.ioc_low.save() - - def setUp(self): - self.client = APIClient() - self.client.force_authenticate(user=self.superuser) - self.url = "/api/feeds/asn/" - - def _get_results(self, response): - payload = response.json() - self.assertIsInstance(payload, list) - return payload - - def test_200_asn_feed_aggregated_fields(self): - """Ensure aggregated fields are computed correctly per ASN using dynamic sums""" - response = self.client.get(self.url) - self.assertEqual(response.status_code, 200) - results = self._get_results(response) - - # filtering high ASN - high_item = next((item for item in results if str(item["asn"]) == self.high_asn), None) - self.assertIsNotNone(high_item) - - # getting all IOCs for high ASN from the DB - high_iocs = IOC.objects.filter(asn=self.high_asn) - - self.assertEqual(high_item["ioc_count"], high_iocs.count()) - self.assertEqual(high_item["total_attack_count"], sum(i.attack_count for i in high_iocs)) - self.assertEqual(high_item["total_interaction_count"], sum(i.interaction_count for i in high_iocs)) - self.assertEqual(high_item["total_login_attempts"], sum(i.login_attempts for i in high_iocs)) - self.assertAlmostEqual(high_item["expected_ioc_count"], sum(i.recurrence_probability for i in high_iocs)) - self.assertAlmostEqual(high_item["expected_interactions"], sum(i.expected_interactions for i in high_iocs)) - - # validating first_seen / last_seen dynamically - self.assertEqual(high_item["first_seen"], min(i.first_seen for i in high_iocs).isoformat()) - self.assertEqual(high_item["last_seen"], max(i.last_seen for i in high_iocs).isoformat()) - - # validating honeypots dynamically - expected_honeypots = sorted({hp.name for i in high_iocs for hp in i.general_honeypot.all()}) - self.assertEqual(sorted(high_item["honeypots"]), expected_honeypots) - - def test_200_asn_feed_default_ordering(self): - response = self.client.get(self.url) - self.assertEqual(response.status_code, 200) - results = self._get_results(response) - - # high_asn has ioc_count=2 > low_asn ioc_count=1 - self.assertEqual(str(results[0]["asn"]), self.high_asn) - self.assertEqual(str(results[1]["asn"]), self.low_asn) - - def test_200_asn_feed_ordering_desc_ioc_count(self): - response = self.client.get(self.url + "?ordering=-ioc_count") - self.assertEqual(response.status_code, 200) - results = self._get_results(response) - - self.assertEqual(str(results[0]["asn"]), self.high_asn) - - def test_200_asn_feed_ordering_asc_ioc_count(self): - response = self.client.get(self.url + "?ordering=ioc_count") - self.assertEqual(response.status_code, 200) - results = self._get_results(response) - self.assertEqual(str(results[0]["asn"]), self.low_asn) - - def test_200_asn_feed_ordering_desc_interaction_count(self): - response = self.client.get(self.url + "?ordering=-total_interaction_count") - self.assertEqual(response.status_code, 200) - results = self._get_results(response) - self.assertEqual(str(results[0]["asn"]), self.high_asn) - - def test_200_asn_feed_with_asn_filter(self): - response = self.client.get(self.url + f"?asn={self.high_asn}") - self.assertEqual(response.status_code, 200) - - results = self._get_results(response) - self.assertEqual(len(results), 1) - self.assertEqual(str(results[0]["asn"]), self.high_asn) - - def test_400_asn_feed_invalid_ordering_honeypots(self): - response = self.client.get(self.url + "?ordering=honeypots") - self.assertEqual(response.status_code, 400) - data = response.json() - errors_container = data.get("errors", data) - error_list = errors_container.get("ordering", []) - self.assertTrue(error_list) - error_msg = error_list[0].lower() - self.assertIn("honeypots", error_msg) - self.assertIn("invalid", error_msg) - - def test_400_asn_feed_invalid_ordering_random(self): - response = self.client.get(self.url + "?ordering=xyz123") - self.assertEqual(response.status_code, 400) - data = response.json() - errors_container = data.get("errors", data) - error_list = errors_container.get("ordering", []) - self.assertTrue(error_list) - error_msg = error_list[0].lower() - self.assertIn("xyz123", error_msg) - self.assertIn("invalid", error_msg) - - def test_400_asn_feed_invalid_ordering_model_field_not_in_agg(self): - response = self.client.get(self.url + "?ordering=attack_count") - self.assertEqual(response.status_code, 400) - data = response.json() - errors_container = data.get("errors", data) - error_list = errors_container.get("ordering", []) - self.assertTrue(error_list) - error_msg = error_list[0].lower() - self.assertIn("attack_count", error_msg) - self.assertIn("invalid", error_msg) - - def test_400_asn_feed_ordering_empty_param(self): - response = self.client.get(self.url + "?ordering=") - self.assertEqual(response.status_code, 400) - data = response.json() - errors_container = data.get("errors", data) - error_list = errors_container.get("ordering", []) - self.assertTrue(error_list) - error_msg = error_list[0].lower() - self.assertIn("blank", error_msg) - - def test_asn_feed_ignores_feed_size(self): - response = self.client.get(self.url + "?feed_size=1") - results = response.json() - # aggregation should return all ASNs regardless of feed_size - self.assertEqual(len(results), 2) - - -class StatisticsViewTestCase(CustomTestCase): - @classmethod - def setUpClass(cls): - super().setUpClass() - Statistics.objects.all().delete() - Statistics.objects.create(source="140.246.171.141", view=ViewType.FEEDS_VIEW.value) - Statistics.objects.create(source="140.246.171.141", view=ViewType.ENRICHMENT_VIEW.value) - - @classmethod - def tearDownClass(cls): - super().tearDownClass() - Statistics.objects.all().delete() - - def test_200_feeds_sources(self): - response = self.client.get("/api/statistics/sources/feeds") - self.assertEqual(response.status_code, 200) - self.assertEqual(response.json()[0]["Sources"], 1) - - def test_200_feeds_downloads(self): - response = self.client.get("/api/statistics/downloads/feeds") - self.assertEqual(response.status_code, 200) - self.assertEqual(response.json()[0]["Downloads"], 1) - - def test_200_enrichment_sources(self): - response = self.client.get("/api/statistics/sources/enrichment") - self.assertEqual(response.status_code, 200) - self.assertEqual(response.json()[0]["Sources"], 1) - - def test_200_enrichment_requests(self): - response = self.client.get("/api/statistics/requests/enrichment") - self.assertEqual(response.status_code, 200) - self.assertEqual(response.json()[0]["Requests"], 1) - - def test_200_feed_types(self): - # Count honeypots before adding new one - initial_count = GeneralHoneypot.objects.count() - # add a general honeypot without associated ioc - GeneralHoneypot(name="Tanner", active=True).save() - self.assertEqual(GeneralHoneypot.objects.count(), initial_count + 1) - - response = self.client.get("/api/statistics/feeds_types") - self.assertEqual(response.status_code, 200) - # Expecting 3 because setupTestData creates 3 IOCs (ioc, ioc_2, ioc_domain) associated with Heralding - self.assertEqual(response.json()[0]["Heralding"], 3) - self.assertEqual(response.json()[0]["Ciscoasa"], 2) - self.assertEqual(response.json()[0]["Log4pot"], 3) - self.assertEqual(response.json()[0]["Cowrie"], 3) - self.assertEqual(response.json()[0]["Tanner"], 0) - - -class GeneralHoneypotViewTestCase(CustomTestCase): - def test_200_all_general_honeypots(self): - initial_count = GeneralHoneypot.objects.count() - # add a general honeypot not active - GeneralHoneypot(name="Adbhoney", active=False).save() - self.assertEqual(GeneralHoneypot.objects.count(), initial_count + 1) - - response = self.client.get("/api/general_honeypot") - self.assertEqual(response.status_code, 200) - # Verify the newly created honeypot is in the response - self.assertIn("Adbhoney", response.json()) - - def test_200_active_general_honeypots(self): - response = self.client.get("/api/general_honeypot?onlyActive=true") - self.assertEqual(response.status_code, 200) - result = response.json() - # Should include active honeypots from CustomTestCase - self.assertIn("Heralding", result) - self.assertIn("Ciscoasa", result) - # Should NOT include inactive honeypot - self.assertNotIn("Ddospot", result) - - -class CommandSequenceViewTestCase(CustomTestCase): - """Test cases for the command_sequence_view.""" - - def setUp(self): - # setup client - self.client = APIClient() - self.client.force_authenticate(user=self.superuser) - - def test_missing_query_parameter(self): - """Test that view returns BadRequest when query parameter is missing.""" - response = self.client.get("/api/command_sequence") - self.assertEqual(response.status_code, 400) - - def test_invalid_query_parameter(self): - """Test that view returns BadRequest when query parameter is invalid.""" - response = self.client.get("/api/command_sequence?query=invalid-input}") - self.assertEqual(response.status_code, 400) - - def test_ip_address_query(self): - """Test view with a valid IP address query.""" - response = self.client.get("/api/command_sequence?query=140.246.171.141") - self.assertEqual(response.status_code, 200) - self.assertIn("executed_commands", response.data) - self.assertIn("executed_by", response.data) - - def test_ip_address_query_with_similar(self): - """Test view with a valid IP address query including similar sequences.""" - response = self.client.get("/api/command_sequence?query=140.246.171.141&include_similar") - self.assertEqual(response.status_code, 200) - self.assertIn("executed_commands", response.data) - self.assertIn("executed_by", response.data) - - def test_nonexistent_ip_address(self): - """Test that view returns 404 for IP with no sequences.""" - response = self.client.get("/api/command_sequence?query=10.0.0.1") - self.assertEqual(response.status_code, 404) - - def test_hash_query(self): - """Test view with a valid hash query.""" - response = self.client.get(f"/api/command_sequence?query={self.hash}") - self.assertEqual(response.status_code, 200) - self.assertIn("commands", response.data) - self.assertIn("iocs", response.data) - - def test_hash_query_with_similar(self): - """Test view with a valid hash query including similar sequences.""" - response = self.client.get(f"/api/command_sequence?query={self.hash}&include_similar") - self.assertEqual(response.status_code, 200) - self.assertIn("commands", response.data) - self.assertIn("iocs", response.data) - - def test_nonexistent_hash(self): - """Test that view returns 404 for nonexistent hash.""" - response = self.client.get(f"/api/command_sequence?query={'f' * 64}") - self.assertEqual(response.status_code, 404) - - @override_settings(FEEDS_LICENSE="https://example.com/license") - def test_ip_address_query_with_license(self): - """Test that license is included when FEEDS_LICENSE is populated.""" - response = self.client.get("/api/command_sequence?query=140.246.171.141") - self.assertEqual(response.status_code, 200) - self.assertIn("license", response.data) - self.assertEqual(response.data["license"], "https://example.com/license") - - @override_settings(FEEDS_LICENSE="") - def test_ip_address_query_without_license(self): - """Test that license is not included when FEEDS_LICENSE is empty.""" - response = self.client.get("/api/command_sequence?query=140.246.171.141") - self.assertEqual(response.status_code, 200) - self.assertNotIn("license", response.data) - - @override_settings(FEEDS_LICENSE="https://example.com/license") - def test_hash_query_with_license(self): - """Test that license is included when FEEDS_LICENSE is populated.""" - response = self.client.get(f"/api/command_sequence?query={self.hash}") - self.assertEqual(response.status_code, 200) - self.assertIn("license", response.data) - self.assertEqual(response.data["license"], "https://example.com/license") - - @override_settings(FEEDS_LICENSE="") - def test_hash_query_without_license(self): - """Test that license is not included when FEEDS_LICENSE is empty.""" - response = self.client.get(f"/api/command_sequence?query={self.hash}") - self.assertEqual(response.status_code, 200) - self.assertNotIn("license", response.data) - - -class CowrieSessionViewTestCase(CustomTestCase): - """Test cases for the cowrie_session_view.""" - - def setUp(self): - # setup client - self.client = APIClient() - self.client.force_authenticate(user=self.superuser) - - # # # # # Basic IP Query Test # # # # # - def test_ip_address_query(self): - """Test view with a valid IP address query.""" - response = self.client.get("/api/cowrie_session?query=140.246.171.141") - self.assertEqual(response.status_code, 200) - self.assertIn("query", response.data) - self.assertIn("commands", response.data) - self.assertIn("sources", response.data) - self.assertNotIn("credentials", response.data) - self.assertNotIn("sessions", response.data) - - def test_ip_address_query_with_similar(self): - """Test view with a valid IP address query including similar sequences.""" - response = self.client.get("/api/cowrie_session?query=140.246.171.141&include_similar=true") - self.assertEqual(response.status_code, 200) - self.assertIn("query", response.data) - self.assertIn("commands", response.data) - self.assertIn("sources", response.data) - self.assertNotIn("credentials", response.data) - self.assertNotIn("sessions", response.data) - self.assertEqual(len(response.data["sources"]), 2) - - def test_ip_address_query_with_credentials(self): - """Test view with a valid IP address query including credentials.""" - response = self.client.get("/api/cowrie_session?query=140.246.171.141&include_credentials=true") - self.assertEqual(response.status_code, 200) - self.assertIn("query", response.data) - self.assertIn("commands", response.data) - self.assertIn("sources", response.data) - self.assertIn("credentials", response.data) - self.assertNotIn("sessions", response.data) - self.assertEqual(len(response.data["credentials"]), 1) - self.assertEqual(response.data["credentials"][0], "root | root") - - def test_ip_address_query_with_sessions(self): - """Test view with a valid IP address query including session data.""" - response = self.client.get("/api/cowrie_session?query=140.246.171.141&include_session_data=true") - self.assertEqual(response.status_code, 200) - self.assertIn("query", response.data) - self.assertIn("commands", response.data) - self.assertIn("sources", response.data) - self.assertNotIn("credentials", response.data) - self.assertIn("sessions", response.data) - self.assertEqual(len(response.data["sessions"]), 1) - self.assertIn("time", response.data["sessions"][0]) - self.assertEqual(response.data["sessions"][0]["duration"], 1.234) - self.assertEqual(response.data["sessions"][0]["source"], "140.246.171.141") - self.assertEqual(response.data["sessions"][0]["interactions"], 5) - self.assertEqual(response.data["sessions"][0]["credentials"][0], "root | root") - self.assertEqual(response.data["sessions"][0]["commands"], "cd foo\nls -la") - - def test_ip_address_query_with_all(self): - """Test view with a valid IP address query including everything.""" - response = self.client.get("/api/cowrie_session?query=140.246.171.141&include_similar=true&include_credentials=true&include_session_data=true") - self.assertEqual(response.status_code, 200) - self.assertIn("query", response.data) - self.assertIn("commands", response.data) - self.assertIn("sources", response.data) - self.assertIn("credentials", response.data) - self.assertIn("sessions", response.data) - - # # # # # Basic Hash Query Test # # # # # - def test_hash_query(self): - """Test view with a valid hash query.""" - response = self.client.get(f"/api/cowrie_session?query={self.hash}") - self.assertEqual(response.status_code, 200) - self.assertIn("query", response.data) - self.assertIn("commands", response.data) - self.assertIn("sources", response.data) - self.assertNotIn("credentials", response.data) - self.assertNotIn("sessions", response.data) - - def test_hash_query_with_all(self): - """Test view with a valid hash query including everything.""" - response = self.client.get(f"/api/cowrie_session?query={self.hash}&include_similar=true&include_credentials=true&include_session_data=true") - self.assertEqual(response.status_code, 200) - self.assertIn("query", response.data) - self.assertIn("commands", response.data) - self.assertIn("sources", response.data) - self.assertIn("credentials", response.data) - self.assertIn("sessions", response.data) - self.assertEqual(len(response.data["sources"]), 2) - - # # # # # IP Address Validation Tests # # # # # - def test_nonexistent_ip_address(self): - """Test that view returns 404 for IP with no sequences.""" - response = self.client.get("/api/cowrie_session?query=10.0.0.1") - self.assertEqual(response.status_code, 404) - - def test_ipv6_address_query(self): - """Test view with a valid IPv6 address query.""" - response = self.client.get("/api/cowrie_session?query=2001:db8::1") - self.assertEqual(response.status_code, 404) - - def test_invalid_ip_format(self): - """Test that malformed IP addresses are rejected.""" - response = self.client.get("/api/cowrie_session?query=999.999.999.999") - self.assertEqual(response.status_code, 400) - - def test_ip_with_cidr_notation(self): - """Test that CIDR notation is rejected.""" - response = self.client.get("/api/cowrie_session?query=192.168.1.0/24") - self.assertEqual(response.status_code, 400) - - # # # # # Parameter Validation Tests # # # # # - def test_missing_query_parameter(self): - """Test that view returns BadRequest when query parameter is missing.""" - response = self.client.get("/api/cowrie_session") - self.assertEqual(response.status_code, 400) - - def test_invalid_query_parameter(self): - """Test that view returns BadRequest when query parameter is invalid.""" - response = self.client.get("/api/cowrie_session?query=invalid-input}") - self.assertEqual(response.status_code, 400) - - def test_include_credentials_invalid_value(self): - """Test that invalid boolean values default to false.""" - response = self.client.get("/api/cowrie_session?query=140.246.171.141&include_credentials=maybe") - self.assertEqual(response.status_code, 200) - self.assertNotIn("credentials", response.data) - - def test_case_insensitive_boolean_parameters(self): - """Test that boolean parameters accept various case formats.""" - response = self.client.get("/api/cowrie_session?query=140.246.171.141&include_credentials=TRUE") - self.assertEqual(response.status_code, 200) - self.assertIn("credentials", response.data) - - response = self.client.get("/api/cowrie_session?query=140.246.171.141&include_credentials=True") - self.assertEqual(response.status_code, 200) - self.assertIn("credentials", response.data) - - # # # # # Hash Validation Tests # # # # # - def test_nonexistent_hash(self): - """Test that view returns 404 for nonexistent hash.""" - response = self.client.get(f"/api/cowrie_session?query={'f' * 64}") - self.assertEqual(response.status_code, 404) - - def test_hash_wrong_length(self): - """Test that hashes with incorrect length are rejected.""" - response = self.client.get("/api/cowrie_session?query=" + "a" * 32) # 32 chars instead of 64 - self.assertEqual(response.status_code, 400) - - def test_hash_invalid_characters(self): - """Test that hashes with invalid characters are rejected.""" - invalid_hash = "g" * 64 # 'g' is not a valid hex character - response = self.client.get(f"/api/cowrie_session?query={invalid_hash}") - self.assertEqual(response.status_code, 400) - - def test_hash_case_insensitive(self): - """Test that hash queries are case-insensitive.""" - response_lower = self.client.get(f"/api/cowrie_session?query={self.hash.lower()}") - response_upper = self.client.get(f"/api/cowrie_session?query={self.hash.upper()}") - self.assertEqual(response_lower.status_code, response_upper.status_code) - - # # # # # Special Characters & Encoding Tests # # # # # - def test_query_with_url_encoding(self): - """Test that URL-encoded queries work correctly.""" - response = self.client.get("/api/cowrie_session?query=140.246.171.141%20") - # Should either work or return 400, not crash - self.assertIn(response.status_code, [200, 400, 404]) - - # # # # # License Tests # # # # # - @override_settings(FEEDS_LICENSE="https://example.com/license") - def test_ip_query_with_license(self): - """Test that license is included when FEEDS_LICENSE is populated.""" - response = self.client.get("/api/cowrie_session?query=140.246.171.141") - self.assertEqual(response.status_code, 200) - self.assertIn("license", response.data) - self.assertEqual(response.data["license"], "https://example.com/license") - - @override_settings(FEEDS_LICENSE="") - def test_ip_query_without_license(self): - """Test that license is not included when FEEDS_LICENSE is empty.""" - response = self.client.get("/api/cowrie_session?query=140.246.171.141") - self.assertEqual(response.status_code, 200) - self.assertNotIn("license", response.data) - - @override_settings(FEEDS_LICENSE="https://example.com/license") - def test_hash_query_with_license(self): - """Test that license is included when FEEDS_LICENSE is populated.""" - response = self.client.get(f"/api/cowrie_session?query={self.hash}") - self.assertEqual(response.status_code, 200) - self.assertIn("license", response.data) - self.assertEqual(response.data["license"], "https://example.com/license") - - @override_settings(FEEDS_LICENSE="") - def test_hash_query_without_license(self): - """Test that license is not included when FEEDS_LICENSE is empty.""" - response = self.client.get(f"/api/cowrie_session?query={self.hash}") - self.assertEqual(response.status_code, 200) - self.assertNotIn("license", response.data) - - def test_query_with_special_characters(self): - """Test handling of queries with special characters.""" - response = self.client.get("/api/cowrie_session?query=") - self.assertEqual(response.status_code, 400) - - # # # # # Authentication & Authorization Tests # # # # # - def test_unauthenticated_request(self): - """Test that unauthenticated requests are rejected.""" - client = APIClient() # No authentication - response = client.get("/api/cowrie_session?query=140.246.171.141") - self.assertEqual(response.status_code, 401) - - def test_regular_user_access(self): - """Test that regular (non-superuser) authenticated users can access.""" - client = APIClient() - client.force_authenticate(user=self.regular_user) - response = client.get("/api/cowrie_session?query=140.246.171.141") - self.assertEqual(response.status_code, 200) - - -class ValidationHelpersTestCase(CustomTestCase): - """Test cases for the validation helper functions.""" - - def test_is_ip_address_valid_ipv4(self): - """Test that is_ip_address returns True for valid IPv4 addresses.""" - self.assertTrue(is_ip_address("192.168.1.1")) - self.assertTrue(is_ip_address("10.0.0.1")) - self.assertTrue(is_ip_address("127.0.0.1")) - - def test_is_ip_address_valid_ipv6(self): - """Test that is_ip_address returns True for valid IPv6 addresses.""" - self.assertTrue(is_ip_address("::1")) - self.assertTrue(is_ip_address("2001:db8::1")) - self.assertTrue(is_ip_address("fe80::1ff:fe23:4567:890a")) - - def test_is_ip_address_invalid(self): - """Test that is_ip_address returns False for invalid IP addresses.""" - self.assertFalse(is_ip_address("not-an-ip")) - self.assertFalse(is_ip_address("256.256.256.256")) - self.assertFalse(is_ip_address("192.168.0")) - self.assertFalse(is_ip_address("2001:xyz::1")) - - def test_is_sha256hash_valid(self): - """Test that is_sha256hash returns True for valid SHA-256 hashes.""" - self.assertTrue(is_sha256hash("a" * 64)) - self.assertTrue(is_sha256hash("1234567890abcdef" * 4)) - self.assertTrue(is_sha256hash("A" * 64)) - - def test_is_sha256hash_invalid(self): - """Test that is_sha256hash returns False for invalid SHA-256 hashes.""" - self.assertFalse(is_sha256hash("a" * 63)) # Too short - self.assertFalse(is_sha256hash("a" * 65)) # Too long - self.assertFalse(is_sha256hash("z" * 64)) # Invalid chars - self.assertFalse(is_sha256hash("not-a-hash"))