diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 3a74cf4a..cba1464f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -9,6 +9,8 @@ updates: interval: "weekly" day: "friday" time: "00:00" + cooldown: + days: 7 # Maintain dependencies for Github Actions - package-ecosystem: "github-actions" directory: "/" @@ -20,3 +22,5 @@ updates: github: patterns: - "actions/*" + cooldown: + days: 7 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9067674f..8a18c0a3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,23 +3,21 @@ repos: rev: v2.2.3 hooks: - id: trailing-whitespace - - id: flake8 - id: check-yaml - id: check-added-large-files exclude: 'uv.lock' - - id: debug-statements - id: end-of-file-fixer exclude: '^.+?\.json$' - - repo: https://github.com/PyCQA/flake8 - rev: 7.1.2 + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.15.8 hooks: - - id: flake8 - additional_dependencies: ['flake8-print==5.0.0'] - - repo: https://github.com/asottile/reorder_python_imports - rev: v1.4.0 + - id: ruff + args: [--fix] + - id: ruff-format + - repo: https://github.com/astral-sh/uv-pre-commit + rev: 0.11.3 hooks: - - id: reorder-python-imports - language_version: python3 + - id: uv-lock - repo: https://github.com/google/yamlfmt rev: v0.14.0 hooks: diff --git a/codecov.yml b/codecov.yml index 2b5037ed..d358f5da 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,17 +1,14 @@ codecov: notify: require_ci_to_pass: yes - coverage: precision: 2 round: down range: "70...100" - status: project: yes patch: yes changes: no - parsers: gcov: branch_detection: @@ -19,7 +16,6 @@ parsers: loop: yes method: no macro: no - comment: layout: "header, diff" behavior: default diff --git a/docs/deletion/index.rst b/docs/deletion/index.rst index 782a3fca..984f5df7 100644 --- a/docs/deletion/index.rst +++ b/docs/deletion/index.rst @@ -19,4 +19,3 @@ There are times, such as GDPR removal requests, when it's necessary to actually This is handled using a ``HardDeletedModels`` table. Subclasses of ``SyncableModel`` should override the ``delete`` method to take a ``hard_delete`` boolean, and add the record to the ``HardDeletedModels`` table when this is passed. On serialization, Morango clears the ``serialized`` field entry in the store for records in ``HardDeletedModels`` and turns on the ``hard_deleted`` flag. Upon syncing with other Morango instances, the hard deletion will propagate to the store record of other instances. - diff --git a/docs/merging/index.rst b/docs/merging/index.rst index fdf98630..edd322c6 100644 --- a/docs/merging/index.rst +++ b/docs/merging/index.rst @@ -36,4 +36,3 @@ In the illustration above: 5. When *Device A* syncs data with *Device B* again (the arrow), there is a conflict because both devices have modified ``r``. It is up to the implementing application to determine what the merge conflict resolution strategy is. - diff --git a/morango/__init__.py b/morango/__init__.py index 7e9101ce..f0620fe7 100644 --- a/morango/__init__.py +++ b/morango/__init__.py @@ -1,5 +1,6 @@ try: from importlib.metadata import version + __version__ = version("morango") except ImportError: # Fallback for older Python versions or when package is not installed diff --git a/morango/api/permissions.py b/morango/api/permissions.py index 2ec315e3..ecc7bc25 100644 --- a/morango/api/permissions.py +++ b/morango/api/permissions.py @@ -1,10 +1,7 @@ import json -from django.contrib.auth import authenticate -from django.contrib.auth import get_user_model -from rest_framework import authentication -from rest_framework import exceptions -from rest_framework import permissions +from django.contrib.auth import authenticate, get_user_model +from rest_framework import authentication, exceptions, permissions from morango.models.core import TransferSession from morango.utils import SETTINGS @@ -61,11 +58,7 @@ def has_permission(self, request, view): if hasattr(request.user, "has_morango_certificate_scope_permission"): scope_definition_id = request.data.get("scope_definition") scope_params = json.loads(request.data.get("scope_params")) - if ( - scope_definition_id - and scope_params - and isinstance(scope_params, dict) - ): + if scope_definition_id and scope_params and isinstance(scope_params, dict): return request.user.has_morango_certificate_scope_permission( scope_definition_id, scope_params ) @@ -89,9 +82,7 @@ def has_permission(self, request, view): sesh_id = request.query_params.get("transfer_session_id") if not sesh_id: return False - if not TransferSession.objects.filter( - id=sesh_id, active=True, push=False - ).exists(): + if not TransferSession.objects.filter(id=sesh_id, active=True, push=False).exists(): return False return True diff --git a/morango/api/serializers.py b/morango/api/serializers.py index be92121f..658ed198 100644 --- a/morango/api/serializers.py +++ b/morango/api/serializers.py @@ -1,21 +1,21 @@ -from rest_framework import exceptions -from rest_framework import serializers +from rest_framework import exceptions, serializers from rest_framework.fields import ReadOnlyField from ..models.certificates import Nonce -from ..models.core import Buffer -from ..models.core import Certificate -from ..models.core import InstanceIDModel -from ..models.core import RecordMaxCounterBuffer -from ..models.core import SyncSession -from ..models.core import TransferSession +from ..models.core import ( + Buffer, + Certificate, + InstanceIDModel, + RecordMaxCounterBuffer, + SyncSession, + TransferSession, +) from ..models.fields.crypto import SharedKey from ..utils import SETTINGS from .fields import PublicKeyField class CertificateSerializer(serializers.ModelSerializer): - public_key = PublicKeyField() def validate_parent(self, parent): @@ -60,8 +60,8 @@ class Meta: class SyncSessionSerializer(serializers.ModelSerializer): - client_instance = serializers.CharField(source='client_instance_json') - server_instance = serializers.CharField(source='server_instance_json') + client_instance = serializers.CharField(source="client_instance_json") + server_instance = serializers.CharField(source="server_instance_json") class Meta: model = SyncSession diff --git a/morango/api/urls.py b/morango/api/urls.py index 328c1d0a..9d391960 100644 --- a/morango/api/urls.py +++ b/morango/api/urls.py @@ -1,24 +1,22 @@ from rest_framework import routers -from .viewsets import BufferViewSet -from .viewsets import CertificateChainViewSet -from .viewsets import CertificateViewSet -from .viewsets import MorangoInfoViewSet -from .viewsets import NonceViewSet -from .viewsets import PublicKeyViewSet -from .viewsets import SyncSessionViewSet -from .viewsets import TransferSessionViewSet +from .viewsets import ( + BufferViewSet, + CertificateChainViewSet, + CertificateViewSet, + MorangoInfoViewSet, + NonceViewSet, + PublicKeyViewSet, + SyncSessionViewSet, + TransferSessionViewSet, +) router = routers.SimpleRouter() router.register(r"certificates", CertificateViewSet, basename="certificates") -router.register( - r"certificatechain", CertificateChainViewSet, basename="certificatechain" -) +router.register(r"certificatechain", CertificateChainViewSet, basename="certificatechain") router.register(r"nonces", NonceViewSet, basename="nonces") router.register(r"syncsessions", SyncSessionViewSet, basename="syncsessions") -router.register( - r"transfersessions", TransferSessionViewSet, basename="transfersessions" -) +router.register(r"transfersessions", TransferSessionViewSet, basename="transfersessions") router.register(r"buffers", BufferViewSet, basename="buffers") router.register(r"morangoinfo", MorangoInfoViewSet, basename="morangoinfo") router.register(r"publickey", PublicKeyViewSet, basename="publickey") diff --git a/morango/api/viewsets.py b/morango/api/viewsets.py index bec91465..fa934f0e 100644 --- a/morango/api/viewsets.py +++ b/morango/api/viewsets.py @@ -6,34 +6,20 @@ from django.core.exceptions import ValidationError from django.utils import timezone from ipware import get_client_ip -from rest_framework import mixins -from rest_framework import pagination -from rest_framework import response -from rest_framework import status -from rest_framework import viewsets +from rest_framework import mixins, pagination, response, status, viewsets from rest_framework.parsers import JSONParser import morango from morango import errors -from morango.api import permissions -from morango.api import serializers -from morango.constants import transfer_stages -from morango.constants import transfer_statuses -from morango.constants.capabilities import ASYNC_OPERATIONS -from morango.constants.capabilities import GZIP_BUFFER_POST +from morango.api import permissions, serializers +from morango.constants import transfer_stages, transfer_statuses +from morango.constants.capabilities import ASYNC_OPERATIONS, GZIP_BUFFER_POST from morango.models import certificates -from morango.models.core import Buffer -from morango.models.core import Certificate -from morango.models.core import InstanceIDModel -from morango.models.core import SyncSession -from morango.models.core import TransferSession +from morango.models.core import Buffer, Certificate, InstanceIDModel, SyncSession, TransferSession from morango.models.fields.crypto import SharedKey from morango.sync.context import LocalSessionContext from morango.sync.controller import SessionController -from morango.utils import _assert -from morango.utils import CAPABILITIES -from morango.utils import parse_capabilities_from_server_request - +from morango.utils import CAPABILITIES, _assert, parse_capabilities_from_server_request if GZIP_BUFFER_POST in CAPABILITIES: from .parsers import GzipParser @@ -83,9 +69,7 @@ def create(self, request): ) # create an in-memory instance of the cert from the serialized data and signature - certificate = Certificate.deserialize( - client_cert["serialized"], client_cert["signature"] - ) + certificate = Certificate.deserialize(client_cert["serialized"], client_cert["signature"]) # check if certificate's public key is in our list of shared keys try: @@ -114,9 +98,7 @@ def create(self, request): return response.Response( { "error_class": e.__class__.__name__, - "error_message": getattr( - e, "message", (getattr(e, "args") or ("",))[0] - ), + "error_message": getattr(e, "message", (getattr(e, "args") or ("",))[0]), }, status=status.HTTP_400_BAD_REQUEST, ) @@ -124,9 +106,7 @@ def create(self, request): # we got this far, and everything looks good, so we can save the certificate certificate.save() - return response.Response( - "Certificate chain has been saved", status=status.HTTP_201_CREATED - ) + return response.Response("Certificate chain has been saved", status=status.HTTP_201_CREATED) class CertificateViewSet( @@ -144,7 +124,6 @@ def create(self, request): serialized_cert = serializers.CertificateSerializer(data=request.data) if serialized_cert.is_valid(): - # inflate the provided data into an actual in-memory certificate certificate = Certificate(**serialized_cert.validated_data) @@ -166,9 +145,7 @@ def create(self, request): return response.Response( { "error_class": e.__class__.__name__, - "error_message": getattr( - e, "message", (getattr(e, "args") or ("",))[0] - ), + "error_message": getattr(e, "message", (getattr(e, "args") or ("",))[0]), }, status=status.HTTP_400_BAD_REQUEST, ) @@ -183,9 +160,7 @@ def create(self, request): ) else: - return response.Response( - serialized_cert.errors, status=status.HTTP_400_BAD_REQUEST - ) + return response.Response(serialized_cert.errors, status=status.HTTP_400_BAD_REQUEST) def get_queryset(self): @@ -198,13 +173,10 @@ def get_queryset(self): base_queryset = base_queryset.filter(profile=params["profile"]) try: - # if specified, filter by primary partition, and only include certs the server owns if "primary_partition" in params: target_cert = base_queryset.get(id=params["primary_partition"]) - return target_cert.get_descendants(include_self=True).exclude( - _private_key=None - ) + return target_cert.get_descendants(include_self=True).exclude(_private_key=None) # if specified, return the certificate chain for a certificate owned by the server if "ancestors_of" in params: @@ -256,12 +228,8 @@ def create(self, request): # attempt to load the requested certificates try: - server_cert = Certificate.objects.get( - id=request.data.get("server_certificate_id") - ) - client_cert = Certificate.objects.get( - id=request.data.get("client_certificate_id") - ) + server_cert = Certificate.objects.get(id=request.data.get("server_certificate_id")) + client_cert = Certificate.objects.get(id=request.data.get("client_certificate_id")) except Certificate.DoesNotExist: return response.Response( "Requested certificate does not exist!", @@ -275,9 +243,7 @@ def create(self, request): ) # check that the nonce/id were properly signed - message = "{nonce}:{id}".format( - nonce=request.data.get("nonce"), id=request.data.get("id") - ) + message = "{nonce}:{id}".format(nonce=request.data.get("nonce"), id=request.data.get("id")) if not client_cert.verify(message, request.data["signature"]): return response.Response( "Client certificate failed to verify signature", @@ -288,9 +254,7 @@ def create(self, request): try: certificates.Nonce.use_nonce(request.data["nonce"]) except errors.MorangoNonceError: - return response.Response( - "Nonce is not valid", status=status.HTTP_403_FORBIDDEN - ) + return response.Response("Nonce is not valid", status=status.HTTP_403_FORBIDDEN) client_instance_json = request.data.get("instance") client_instance_id = None @@ -370,12 +334,16 @@ def create(self, request): # noqa: C901 client_scope = syncsession.client_certificate.get_scope() if is_a_push: if not requested_filter.is_subset_of(client_scope.write_filter): - scope_error_msg = "Client certificate scope does not permit pushing for the requested filter." + scope_error_msg = ( + "Client certificate scope does not permit pushing for the requested filter." + ) if not requested_filter.is_subset_of(server_scope.read_filter): scope_error_msg = "Server certificate scope does not permit receiving pushes for the requested filter." else: if not requested_filter.is_subset_of(client_scope.read_filter): - scope_error_msg = "Client certificate scope does not permit pulling for the requested filter." + scope_error_msg = ( + "Client certificate scope does not permit pulling for the requested filter." + ) if not requested_filter.is_subset_of(server_scope.write_filter): scope_error_msg = "Server certificate scope does not permit responding to pulls for the requested filter." if scope_error_msg: @@ -391,11 +359,7 @@ def create(self, request): # noqa: C901 # If both client and ourselves allow async, we just return accepted status, and the client # should PATCH the transfer_session to the appropriate stage. If not async, we wait until # queuing is complete - to_stage = ( - transfer_stages.INITIALIZING - if self.async_allowed() - else transfer_stages.QUEUING - ) + to_stage = transfer_stages.INITIALIZING if self.async_allowed() else transfer_stages.QUEUING result = session_controller.proceed_to_and_wait_for( to_stage, context=context, max_interval=2 ) @@ -469,9 +433,7 @@ def async_allowed(self): :return: A boolean if async ops are allowed by client and self """ client_capabilities = parse_capabilities_from_server_request(self.request) - return ( - ASYNC_OPERATIONS in client_capabilities and ASYNC_OPERATIONS in CAPABILITIES - ) + return ASYNC_OPERATIONS in client_capabilities and ASYNC_OPERATIONS in CAPABILITIES class BufferViewSet(mixins.ListModelMixin, viewsets.GenericViewSet): @@ -495,12 +457,8 @@ def create(self, request): status=status.HTTP_403_FORBIDDEN, ) - context = LocalSessionContext.from_request( - request, transfer_session=transfer_session - ) - result = session_controller.proceed_to( - transfer_stages.TRANSFERRING, context=context - ) + context = LocalSessionContext.from_request(request, transfer_session=transfer_session) + result = session_controller.proceed_to(transfer_stages.TRANSFERRING, context=context) if result == transfer_statuses.ERRORED: if context.error: diff --git a/morango/apps.py b/morango/apps.py index d3f73fb2..118ff964 100644 --- a/morango/apps.py +++ b/morango/apps.py @@ -1,7 +1,6 @@ from django.apps import AppConfig -from morango.registry import session_middleware -from morango.registry import syncable_models +from morango.registry import session_middleware, syncable_models class MorangoConfig(AppConfig): diff --git a/morango/constants/transfer_stages.py b/morango/constants/transfer_stages.py index a1a4499d..8dda46db 100644 --- a/morango/constants/transfer_stages.py +++ b/morango/constants/transfer_stages.py @@ -1,6 +1,7 @@ """ This module contains constants representing the possible stages of a transfer session. """ + from django.utils.translation import gettext_lazy as _ INITIALIZING = "initializing" diff --git a/morango/constants/transfer_statuses.py b/morango/constants/transfer_statuses.py index f2547f19..903f8c81 100644 --- a/morango/constants/transfer_statuses.py +++ b/morango/constants/transfer_statuses.py @@ -1,6 +1,7 @@ """ This module contains constants representing the possible statuses of a transfer session stage. """ + from django.utils.translation import gettext_lazy as _ PENDING = "pending" diff --git a/morango/management/commands/cleanupsyncs.py b/morango/management/commands/cleanupsyncs.py index 4664589f..fb31a107 100644 --- a/morango/management/commands/cleanupsyncs.py +++ b/morango/management/commands/cleanupsyncs.py @@ -6,9 +6,7 @@ from django.db import transaction from django.utils import timezone -from morango.models import SyncSession -from morango.models import TransferSession - +from morango.models import SyncSession, TransferSession logger = logging.getLogger(__name__) @@ -21,7 +19,7 @@ def add_arguments(self, parser): "--ids", type=lambda ids: ids.split(","), default=None, - help="Comma separated list of SyncSession IDs to filter against" + help="Comma separated list of SyncSession IDs to filter against", ) parser.add_argument( "--expiration", diff --git a/morango/migrations/0001_initial.py b/morango/migrations/0001_initial.py index 5b50e69a..5818b01d 100644 --- a/morango/migrations/0001_initial.py +++ b/morango/migrations/0001_initial.py @@ -2,15 +2,13 @@ # Generated by Django 1.9 on 2017-05-08 23:42 import django.db.models.deletion import django.utils.timezone -from django.db import migrations -from django.db import models +from django.db import migrations, models import morango.models.fields.crypto import morango.models.fields.uuids class Migration(migrations.Migration): - initial = True dependencies = [] @@ -52,7 +50,10 @@ class Migration(migrations.Migration): ("public_key", morango.models.fields.crypto.PublicKeyField()), ("serialized", models.TextField()), ("signature", models.TextField()), - ("private_key", morango.models.fields.crypto.PrivateKeyField(blank=True, null=True)), + ( + "private_key", + morango.models.fields.crypto.PrivateKeyField(blank=True, null=True), + ), ("lft", models.PositiveIntegerField(db_index=True, editable=False)), ("rght", models.PositiveIntegerField(db_index=True, editable=False)), ("tree_id", models.PositiveIntegerField(db_index=True, editable=False)), diff --git a/morango/migrations/0001_squashed_0024_auto_20240129_1757.py b/morango/migrations/0001_squashed_0024_auto_20240129_1757.py index 193fd77a..ef74ef89 100644 --- a/morango/migrations/0001_squashed_0024_auto_20240129_1757.py +++ b/morango/migrations/0001_squashed_0024_auto_20240129_1757.py @@ -2,13 +2,11 @@ import django.db.models.deletion import django.db.models.manager import django.utils.timezone -from django.db import migrations -from django.db import models +from django.db import migrations, models import morango.models.fields.crypto import morango.models.fields.uuids - # morango.migrations.0020_postgres_fix_nullable # is ignored in this squashed migration, as it is replaced by # the original change made via an edit to the prior migration. @@ -18,7 +16,6 @@ class Migration(migrations.Migration): - replaces = [ ("morango", "0001_initial"), ("morango", "0002_auto_20170511_0400"), @@ -97,9 +94,7 @@ class Migration(migrations.Migration): fields=[ ( "id", - morango.models.fields.uuids.UUIDField( - primary_key=True, serialize=False - ), + morango.models.fields.uuids.UUIDField(primary_key=True, serialize=False), ), ("profile", models.CharField(max_length=40)), ], @@ -116,9 +111,7 @@ class Migration(migrations.Migration): ("partition", models.TextField()), ( "id", - morango.models.fields.uuids.UUIDField( - primary_key=True, serialize=False - ), + morango.models.fields.uuids.UUIDField(primary_key=True, serialize=False), ), ("conflicting_serialized_data", models.TextField(blank=True)), ("_self_ref_fk", models.CharField(blank=True, max_length=32)), @@ -147,9 +140,7 @@ class Migration(migrations.Migration): fields=[ ( "id", - morango.models.fields.uuids.UUIDField( - primary_key=True, serialize=False - ), + morango.models.fields.uuids.UUIDField(primary_key=True, serialize=False), ), ( "start_timestamp", @@ -160,7 +151,7 @@ class Migration(migrations.Migration): ( "connection_kind", models.CharField( - choices=[('network', 'Network'), ('disk', 'Disk')], + choices=[("network", "Network"), ("disk", "Disk")], max_length=10, ), ), @@ -188,9 +179,7 @@ class Migration(migrations.Migration): fields=[ ( "id", - morango.models.fields.uuids.UUIDField( - primary_key=True, serialize=False - ), + morango.models.fields.uuids.UUIDField(primary_key=True, serialize=False), ), ("filter", models.TextField()), ("push", models.BooleanField()), @@ -308,9 +297,7 @@ class Migration(migrations.Migration): fields=[ ( "id", - morango.models.fields.uuids.UUIDField( - primary_key=True, serialize=False - ), + morango.models.fields.uuids.UUIDField(primary_key=True, serialize=False), ), ("profile", models.CharField(max_length=20)), ("scope_version", models.IntegerField()), @@ -495,9 +482,7 @@ class Migration(migrations.Migration): fields=[ ( "id", - morango.models.fields.uuids.UUIDField( - primary_key=True, serialize=False - ), + morango.models.fields.uuids.UUIDField(primary_key=True, serialize=False), ), ("profile", models.CharField(max_length=40)), ], @@ -556,9 +541,7 @@ class Migration(migrations.Migration): ), migrations.AddIndex( model_name="store", - index=models.Index( - fields=["partition"], name="idx_morango_store_partition" - ), + index=models.Index(fields=["partition"], name="idx_morango_store_partition"), ), migrations.RenameField( model_name="syncsession", diff --git a/morango/migrations/0002_auto_20170511_0400.py b/morango/migrations/0002_auto_20170511_0400.py index 74756775..b6e20d50 100644 --- a/morango/migrations/0002_auto_20170511_0400.py +++ b/morango/migrations/0002_auto_20170511_0400.py @@ -1,14 +1,12 @@ # -*- coding: utf-8 -*- # Generated by Django 1.9.7 on 2017-05-11 04:00 import django.db.models.manager -from django.db import migrations -from django.db import models +from django.db import migrations, models import morango.models.fields.uuids class Migration(migrations.Migration): - dependencies = [("morango", "0001_initial")] operations = [ diff --git a/morango/migrations/0002_store_idx_morango_deserialize.py b/morango/migrations/0002_store_idx_morango_deserialize.py index d581aa21..f86568c8 100644 --- a/morango/migrations/0002_store_idx_morango_deserialize.py +++ b/morango/migrations/0002_store_idx_morango_deserialize.py @@ -1,10 +1,8 @@ # Generated by Django 3.2.24 on 2024-10-18 23:18 -from django.db import migrations -from django.db import models +from django.db import migrations, models class Migration(migrations.Migration): - dependencies = [ ("morango", "0001_squashed_0024_auto_20240129_1757"), ] diff --git a/morango/migrations/0003_auto_20170519_0543.py b/morango/migrations/0003_auto_20170519_0543.py index f1281584..1e658101 100644 --- a/morango/migrations/0003_auto_20170519_0543.py +++ b/morango/migrations/0003_auto_20170519_0543.py @@ -1,18 +1,14 @@ # -*- coding: utf-8 -*- # Generated by Django 1.9 on 2017-05-19 05:43 -from django.db import migrations -from django.db import models +from django.db import migrations, models class Migration(migrations.Migration): - dependencies = [("morango", "0002_auto_20170511_0400")] operations = [ migrations.AlterModelManagers(name="certificate", managers=[]), - migrations.RenameField( - model_name="scopedefinition", old_name="scope_id", new_name="id" - ), + migrations.RenameField(model_name="scopedefinition", old_name="scope_id", new_name="id"), migrations.AddField( model_name="scopedefinition", name="primary_scope_param_key", diff --git a/morango/migrations/0004_auto_20170520_2112.py b/morango/migrations/0004_auto_20170520_2112.py index 46580a2c..12f41f27 100644 --- a/morango/migrations/0004_auto_20170520_2112.py +++ b/morango/migrations/0004_auto_20170520_2112.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies = [("morango", "0003_auto_20170519_0543")] operations = [ diff --git a/morango/migrations/0005_auto_20170629_2139.py b/morango/migrations/0005_auto_20170629_2139.py index 3004fc73..6d2335e1 100644 --- a/morango/migrations/0005_auto_20170629_2139.py +++ b/morango/migrations/0005_auto_20170629_2139.py @@ -6,7 +6,6 @@ class Migration(migrations.Migration): - dependencies = [("morango", "0004_auto_20170520_2112")] operations = [ diff --git a/morango/migrations/0006_instanceidmodel_system_id.py b/morango/migrations/0006_instanceidmodel_system_id.py index f521dd3d..82ed6268 100644 --- a/morango/migrations/0006_instanceidmodel_system_id.py +++ b/morango/migrations/0006_instanceidmodel_system_id.py @@ -1,11 +1,9 @@ # -*- coding: utf-8 -*- # Generated by Django 1.9 on 2017-06-30 00:15 -from django.db import migrations -from django.db import models +from django.db import migrations, models class Migration(migrations.Migration): - dependencies = [("morango", "0005_auto_20170629_2139")] operations = [ diff --git a/morango/migrations/0007_auto_20171018_1615.py b/morango/migrations/0007_auto_20171018_1615.py index 06625a77..635f57fb 100644 --- a/morango/migrations/0007_auto_20171018_1615.py +++ b/morango/migrations/0007_auto_20171018_1615.py @@ -4,15 +4,13 @@ import django.db.models.deletion import django.utils.timezone -from django.db import migrations -from django.db import models +from django.db import migrations, models from django.utils.timezone import utc import morango.models.fields.uuids class Migration(migrations.Migration): - dependencies = [("morango", "0006_instanceidmodel_system_id")] operations = [ @@ -45,9 +43,7 @@ class Migration(migrations.Migration): old_name="write_scope_def", new_name="write_filter_template", ), - migrations.RenameField( - model_name="transfersession", old_name="incoming", new_name="push" - ), + migrations.RenameField(model_name="transfersession", old_name="incoming", new_name="push"), migrations.RemoveField(model_name="syncsession", name="host"), migrations.RemoveField(model_name="syncsession", name="local_scope"), migrations.RemoveField(model_name="syncsession", name="remote_scope"), diff --git a/morango/migrations/0008_auto_20171114_2217.py b/morango/migrations/0008_auto_20171114_2217.py index 9ee895f8..0826238f 100644 --- a/morango/migrations/0008_auto_20171114_2217.py +++ b/morango/migrations/0008_auto_20171114_2217.py @@ -2,12 +2,10 @@ # Generated by Django 1.9.7 on 2017-11-14 22:17 import django.db.models.deletion import django.db.models.manager -from django.db import migrations -from django.db import models +from django.db import migrations, models class Migration(migrations.Migration): - dependencies = [("morango", "0007_auto_20171018_1615")] operations = [ @@ -16,9 +14,7 @@ class Migration(migrations.Migration): old_name="local_instance", new_name="client_instance", ), - migrations.RenameField( - model_name="syncsession", old_name="local_ip", new_name="client_ip" - ), + migrations.RenameField(model_name="syncsession", old_name="local_ip", new_name="client_ip"), migrations.RenameField( model_name="syncsession", old_name="remote_instance", diff --git a/morango/migrations/0009_auto_20171205_0252.py b/morango/migrations/0009_auto_20171205_0252.py index 9e22eb6a..574d1459 100644 --- a/morango/migrations/0009_auto_20171205_0252.py +++ b/morango/migrations/0009_auto_20171205_0252.py @@ -4,7 +4,6 @@ class Migration(migrations.Migration): - dependencies = [("morango", "0008_auto_20171114_2217")] operations = [ diff --git a/morango/migrations/0010_auto_20171206_1615.py b/morango/migrations/0010_auto_20171206_1615.py index 276d5ef7..d61dff7b 100644 --- a/morango/migrations/0010_auto_20171206_1615.py +++ b/morango/migrations/0010_auto_20171206_1615.py @@ -1,11 +1,9 @@ # -*- coding: utf-8 -*- # Generated by Django 1.9.13 on 2017-12-06 22:15 -from django.db import migrations -from django.db import models +from django.db import migrations, models class Migration(migrations.Migration): - dependencies = [("morango", "0009_auto_20171205_0252")] operations = [ diff --git a/morango/migrations/0011_sharedkey.py b/morango/migrations/0011_sharedkey.py index 07761d42..7daeb18e 100644 --- a/morango/migrations/0011_sharedkey.py +++ b/morango/migrations/0011_sharedkey.py @@ -1,13 +1,11 @@ # -*- coding: utf-8 -*- # Generated by Django 1.11.13 on 2018-06-12 18:38 -from django.db import migrations -from django.db import models +from django.db import migrations, models import morango.models.fields.crypto class Migration(migrations.Migration): - dependencies = [("morango", "0010_auto_20171206_1615")] operations = [ diff --git a/morango/migrations/0012_auto_20180927_1658.py b/morango/migrations/0012_auto_20180927_1658.py index f6ff5968..26b8c34f 100644 --- a/morango/migrations/0012_auto_20180927_1658.py +++ b/morango/migrations/0012_auto_20180927_1658.py @@ -1,13 +1,11 @@ # -*- coding: utf-8 -*- # Generated by Django 1.11.15 on 2018-09-27 16:58 -from django.db import migrations -from django.db import models +from django.db import migrations, models import morango.models.fields.uuids class Migration(migrations.Migration): - dependencies = [("morango", "0011_sharedkey")] operations = [ diff --git a/morango/migrations/0013_auto_20190627_1513.py b/morango/migrations/0013_auto_20190627_1513.py index e7f8457e..02f56726 100644 --- a/morango/migrations/0013_auto_20190627_1513.py +++ b/morango/migrations/0013_auto_20190627_1513.py @@ -6,7 +6,6 @@ class Migration(migrations.Migration): - dependencies = [("morango", "0012_auto_20180927_1658")] operations = [ diff --git a/morango/migrations/0014_syncsession_extra_fields.py b/morango/migrations/0014_syncsession_extra_fields.py index d0d25424..ab4b1874 100644 --- a/morango/migrations/0014_syncsession_extra_fields.py +++ b/morango/migrations/0014_syncsession_extra_fields.py @@ -1,11 +1,9 @@ # -*- coding: utf-8 -*- # Generated by Django 1.11.27 on 2019-12-30 18:28 -from django.db import migrations -from django.db import models +from django.db import migrations, models class Migration(migrations.Migration): - dependencies = [ ("morango", "0013_auto_20190627_1513"), ] diff --git a/morango/migrations/0015_auto_20200508_2104.py b/morango/migrations/0015_auto_20200508_2104.py index 326d9738..383a12e9 100644 --- a/morango/migrations/0015_auto_20200508_2104.py +++ b/morango/migrations/0015_auto_20200508_2104.py @@ -1,11 +1,9 @@ # -*- coding: utf-8 -*- # Generated by Django 1.11.29 on 2020-05-08 21:04 -from django.db import migrations -from django.db import models +from django.db import migrations, models class Migration(migrations.Migration): - dependencies = [ ("morango", "0014_syncsession_extra_fields"), ] diff --git a/morango/migrations/0016_store_deserialization_error.py b/morango/migrations/0016_store_deserialization_error.py index 3191b458..bc5a9934 100644 --- a/morango/migrations/0016_store_deserialization_error.py +++ b/morango/migrations/0016_store_deserialization_error.py @@ -1,19 +1,17 @@ # -*- coding: utf-8 -*- # Generated by Django 1.11.28 on 2020-06-10 23:48 -from django.db import migrations -from django.db import models +from django.db import migrations, models class Migration(migrations.Migration): - dependencies = [ - ('morango', '0015_auto_20200508_2104'), + ("morango", "0015_auto_20200508_2104"), ] operations = [ migrations.AddField( - model_name='store', - name='deserialization_error', + model_name="store", + name="deserialization_error", field=models.TextField(blank=True), ), ] diff --git a/morango/migrations/0017_store_last_transfer_session_id.py b/morango/migrations/0017_store_last_transfer_session_id.py index c5a73f37..8c31b362 100644 --- a/morango/migrations/0017_store_last_transfer_session_id.py +++ b/morango/migrations/0017_store_last_transfer_session_id.py @@ -6,15 +6,16 @@ class Migration(migrations.Migration): - dependencies = [ - ('morango', '0016_store_deserialization_error'), + ("morango", "0016_store_deserialization_error"), ] operations = [ migrations.AddField( - model_name='store', - name='last_transfer_session_id', - field=morango.models.fields.uuids.UUIDField(blank=True, db_index=True, default=None, null=True), + model_name="store", + name="last_transfer_session_id", + field=morango.models.fields.uuids.UUIDField( + blank=True, db_index=True, default=None, null=True + ), ), ] diff --git a/morango/migrations/0018_auto_20210714_2216.py b/morango/migrations/0018_auto_20210714_2216.py index 51277dc5..b3c3d191 100644 --- a/morango/migrations/0018_auto_20210714_2216.py +++ b/morango/migrations/0018_auto_20210714_2216.py @@ -1,11 +1,9 @@ # -*- coding: utf-8 -*- # Generated by Django 1.11.29 on 2021-07-14 22:16 -from django.db import migrations -from django.db import models +from django.db import migrations, models class Migration(migrations.Migration): - dependencies = [ ("morango", "0017_store_last_transfer_session_id"), ] diff --git a/morango/migrations/0019_auto_20220113_1807.py b/morango/migrations/0019_auto_20220113_1807.py index f187461a..2ed80e3b 100644 --- a/morango/migrations/0019_auto_20220113_1807.py +++ b/morango/migrations/0019_auto_20220113_1807.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- # Generated by Django 1.11.29 on 2022-01-13 18:07 -from django.db import migrations -from django.db import models +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/morango/migrations/0020_postgres_fix_nullable.py b/morango/migrations/0020_postgres_fix_nullable.py index f35db4d1..0e915fe9 100644 --- a/morango/migrations/0020_postgres_fix_nullable.py +++ b/morango/migrations/0020_postgres_fix_nullable.py @@ -6,15 +6,23 @@ def apply(apps, schema_editor): # sqlite does not allow ALTER COLUMN, but also isn't affected by this issue if "postgresql" in schema_editor.connection.vendor: - schema_editor.execute("ALTER TABLE morango_transfersession ALTER COLUMN transfer_stage DROP NOT NULL") - schema_editor.execute("ALTER TABLE morango_transfersession ALTER COLUMN transfer_stage_status DROP NOT NULL") + schema_editor.execute( + "ALTER TABLE morango_transfersession ALTER COLUMN transfer_stage DROP NOT NULL" + ) + schema_editor.execute( + "ALTER TABLE morango_transfersession ALTER COLUMN transfer_stage_status DROP NOT NULL" + ) def revert(apps, schema_editor): # sqlite does not allow ALTER COLUMN, but also isn't affected by this issue if "postgresql" in schema_editor.connection.vendor: - schema_editor.execute("ALTER TABLE morango_transfersession ALTER COLUMN transfer_stage SET NOT NULL") - schema_editor.execute("ALTER TABLE morango_transfersession ALTER COLUMN transfer_stage_status SET NOT NULL") + schema_editor.execute( + "ALTER TABLE morango_transfersession ALTER COLUMN transfer_stage SET NOT NULL" + ) + schema_editor.execute( + "ALTER TABLE morango_transfersession ALTER COLUMN transfer_stage_status SET NOT NULL" + ) class Migration(migrations.Migration): diff --git a/morango/migrations/0021_store_partition_index_create.py b/morango/migrations/0021_store_partition_index_create.py index f96ed79c..a4030b38 100644 --- a/morango/migrations/0021_store_partition_index_create.py +++ b/morango/migrations/0021_store_partition_index_create.py @@ -1,11 +1,9 @@ # -*- coding: utf-8 -*- # Generated by Django 1.11.29 on 2022-04-27 16:59 -from django.db import migrations -from django.db import models +from django.db import migrations, models class ConditionalConcurrentTextPatternIndex(migrations.AddIndex): - def database_forwards(self, app_label, schema_editor, from_state, to_state): if "postgresql" in schema_editor.connection.vendor: # Most of this code is vendored from here: @@ -15,23 +13,27 @@ def database_forwards(self, app_label, schema_editor, from_state, to_state): model = to_state.apps.get_model(app_label, self.model_name) quote_name = schema_editor.quote_name sql_template = "CREATE INDEX CONCURRENTLY IF NOT EXISTS {index_name} ON {table_name} ({columns} text_pattern_ops){extra}" - fields = [model._meta.get_field(field_name) for field_name, _ in self.index.fields_orders] + fields = [ + model._meta.get_field(field_name) for field_name, _ in self.index.fields_orders + ] tablespace_sql = schema_editor._get_index_tablespace_sql(model, fields) quote_name = schema_editor.quote_name columns = [ - ('%s %s' % (quote_name(field.column), order)).strip() + ("%s %s" % (quote_name(field.column), order)).strip() for field, (_, order) in zip(fields, self.index.fields_orders) ] schema_editor.execute( sql_template.format( index_name=quote_name(self.index.name), table_name=quote_name(model._meta.db_table), - columns=', '.join(columns), - extra=tablespace_sql + columns=", ".join(columns), + extra=tablespace_sql, ) ) else: - super(ConditionalConcurrentTextPatternIndex, self).database_forwards(app_label, schema_editor, from_state, to_state) + super(ConditionalConcurrentTextPatternIndex, self).database_forwards( + app_label, schema_editor, from_state, to_state + ) class Migration(migrations.Migration): @@ -39,12 +41,12 @@ class Migration(migrations.Migration): atomic = False dependencies = [ - ('morango', '0020_postgres_fix_nullable'), + ("morango", "0020_postgres_fix_nullable"), ] operations = [ ConditionalConcurrentTextPatternIndex( - model_name='store', - index=models.Index(fields=['partition'], name='idx_morango_store_partition'), + model_name="store", + index=models.Index(fields=["partition"], name="idx_morango_store_partition"), ) ] diff --git a/morango/migrations/0022_rename_instance_fields.py b/morango/migrations/0022_rename_instance_fields.py index 115df24d..d0634e80 100644 --- a/morango/migrations/0022_rename_instance_fields.py +++ b/morango/migrations/0022_rename_instance_fields.py @@ -4,20 +4,19 @@ class Migration(migrations.Migration): - dependencies = [ - ('morango', '0021_store_partition_index_create'), + ("morango", "0021_store_partition_index_create"), ] operations = [ migrations.RenameField( - model_name='syncsession', - old_name='client_instance', - new_name='client_instance_json', + model_name="syncsession", + old_name="client_instance", + new_name="client_instance_json", ), migrations.RenameField( - model_name='syncsession', - old_name='server_instance', - new_name='server_instance_json', + model_name="syncsession", + old_name="server_instance", + new_name="server_instance_json", ), ] diff --git a/morango/migrations/0023_add_instance_id_fields.py b/morango/migrations/0023_add_instance_id_fields.py index bc9f12a5..fb6c707c 100644 --- a/morango/migrations/0023_add_instance_id_fields.py +++ b/morango/migrations/0023_add_instance_id_fields.py @@ -1,24 +1,22 @@ # -*- coding: utf-8 -*- # Generated by Django 1.11.29 on 2023-01-31 19:03 -from django.db import migrations -from django.db import models +from django.db import migrations, models class Migration(migrations.Migration): - dependencies = [ - ('morango', '0022_rename_instance_fields'), + ("morango", "0022_rename_instance_fields"), ] operations = [ migrations.AddField( - model_name='syncsession', - name='client_instance_id', + model_name="syncsession", + name="client_instance_id", field=models.UUIDField(blank=True, null=True), ), migrations.AddField( - model_name='syncsession', - name='server_instance_id', + model_name="syncsession", + name="server_instance_id", field=models.UUIDField(blank=True, null=True), ), ] diff --git a/morango/migrations/0024_auto_20240129_1757.py b/morango/migrations/0024_auto_20240129_1757.py index 99a17a74..e3c66308 100644 --- a/morango/migrations/0024_auto_20240129_1757.py +++ b/morango/migrations/0024_auto_20240129_1757.py @@ -1,28 +1,26 @@ # Generated by Django 3.2.23 on 2024-01-29 17:57 -from django.db import migrations -from django.db import models +from django.db import migrations, models class Migration(migrations.Migration): - dependencies = [ - ('morango', '0023_add_instance_id_fields'), + ("morango", "0023_add_instance_id_fields"), ] operations = [ migrations.AlterField( - model_name='certificate', - name='level', + model_name="certificate", + name="level", field=models.PositiveIntegerField(editable=False), ), migrations.AlterField( - model_name='certificate', - name='lft', + model_name="certificate", + name="lft", field=models.PositiveIntegerField(editable=False), ), migrations.AlterField( - model_name='certificate', - name='rght', + model_name="certificate", + name="rght", field=models.PositiveIntegerField(editable=False), ), ] diff --git a/morango/models/__init__.py b/morango/models/__init__.py index 1121fadc..2e48ad0e 100644 --- a/morango/models/__init__.py +++ b/morango/models/__init__.py @@ -1,21 +1,19 @@ from morango.models import signals -from morango.models.certificates import Certificate -from morango.models.certificates import Filter -from morango.models.certificates import Nonce -from morango.models.certificates import Scope -from morango.models.certificates import ScopeDefinition -from morango.models.core import Buffer -from morango.models.core import DatabaseIDModel -from morango.models.core import DatabaseMaxCounter -from morango.models.core import DeletedModels -from morango.models.core import HardDeletedModels -from morango.models.core import InstanceIDModel -from morango.models.core import RecordMaxCounter -from morango.models.core import RecordMaxCounterBuffer -from morango.models.core import Store -from morango.models.core import SyncableModel -from morango.models.core import SyncSession -from morango.models.core import TransferSession +from morango.models.certificates import Certificate, Filter, Nonce, Scope, ScopeDefinition +from morango.models.core import ( + Buffer, + DatabaseIDModel, + DatabaseMaxCounter, + DeletedModels, + HardDeletedModels, + InstanceIDModel, + RecordMaxCounter, + RecordMaxCounterBuffer, + Store, + SyncableModel, + SyncSession, + TransferSession, +) from morango.models.fields import * # noqa from morango.models.fields import __all__ as fields_all from morango.models.fields.crypto import SharedKey @@ -24,7 +22,6 @@ from morango.models.query import SyncableModelQuerySet from morango.registry import syncable_models - __all__ = fields_all __all__ += [ "SharedKey", diff --git a/morango/models/certificates.py b/morango/models/certificates.py index 1f6cf466..36159d31 100644 --- a/morango/models/certificates.py +++ b/morango/models/certificates.py @@ -3,6 +3,7 @@ Each certificate has a ``private_key`` used for signing (child) certificates (thus giving certain permissions) and a ``public_key`` used for verifying that a certificate(s) was properly signed. """ + import json import logging import string @@ -10,29 +11,27 @@ import mptt.models from django.core.management import call_command -from django.db import connection -from django.db import models -from django.db import transaction +from django.db import connection, models, transaction from django.db.utils import OperationalError from django.utils import timezone -from .fields.crypto import Key -from .fields.crypto import PrivateKeyField -from .fields.crypto import PublicKeyField -from .fields.uuids import UUIDModelMixin -from morango.errors import CertificateIDInvalid -from morango.errors import CertificateProfileInvalid -from morango.errors import CertificateRootScopeInvalid -from morango.errors import CertificateScopeNotSubset -from morango.errors import CertificateSignatureInvalid -from morango.errors import NonceDoesNotExist -from morango.errors import NonceExpired +from morango.errors import ( + CertificateIDInvalid, + CertificateProfileInvalid, + CertificateRootScopeInvalid, + CertificateScopeNotSubset, + CertificateSignatureInvalid, + NonceDoesNotExist, + NonceExpired, +) from morango.sync.backends.utils import load_backend from morango.utils import _assert +from .fields.crypto import Key, PrivateKeyField, PublicKeyField +from .fields.uuids import UUIDModelMixin -class Certificate(mptt.models.MPTTModel, UUIDModelMixin): +class Certificate(mptt.models.MPTTModel, UUIDModelMixin): uuid_input_fields = ("public_key", "profile", "salt") parent = models.ForeignKey("Certificate", blank=True, null=True, on_delete=models.CASCADE) @@ -43,9 +42,7 @@ class Certificate(mptt.models.MPTTModel, UUIDModelMixin): # scope of this certificate, and version of the scope, along with associated params scope_definition = models.ForeignKey("ScopeDefinition", on_delete=models.CASCADE) scope_version = models.IntegerField() - scope_params = ( - models.TextField() - ) # JSON dict of values to insert into scope definitions + scope_params = models.TextField() # JSON dict of values to insert into scope definitions # track the certificate's public key so we can verify any certificates it signs public_key = PublicKeyField() @@ -70,9 +67,7 @@ def private_key(self): def private_key(self, value): self._private_key = value if value and not self.public_key: - self.public_key = Key( - public_key_string=self._private_key.get_public_key_string() - ) + self.public_key = Key(public_key_string=self._private_key.get_public_key_string()) @classmethod def generate_root_certificate(cls, scope_def_id, **extra_scope_params): @@ -95,9 +90,7 @@ def generate_root_certificate(cls, scope_def_id, **extra_scope_params): # generate a key and extract the public key component cert.private_key = Key() - cert.public_key = Key( - public_key_string=cert.private_key.get_public_key_string() - ) + cert.public_key = Key(public_key_string=cert.private_key.get_public_key_string()) # calculate the certificate's ID on the basis of the profile and public key cert.id = cert.calculate_uuid() @@ -162,9 +155,7 @@ def check_certificate(self): # check that the certificate's ID is properly calculated if self.id != self.calculate_uuid(): raise CertificateIDInvalid( - "Certificate ID is {} but should be {}".format( - self.id, self.calculate_uuid() - ) + "Certificate ID is {} but should be {}".format(self.id, self.calculate_uuid()) ) if not self.parent: # self-signed root certificate @@ -240,9 +231,7 @@ def save_certificate_chain(cls, cert_chain, expected_last_id=None): return cert def sign(self, value): - _assert( - self.private_key, "Can only sign using certificates that have private keys" - ) + _assert(self.private_key, "Can only sign using certificates that have private keys") return self.private_key.sign(value) def verify(self, value, signature): @@ -272,7 +261,9 @@ def _lock_mptt(self): yield except OperationalError as e: if "deadlock detected" in e.args[0]: - logging.error("Deadlock detected when attempting to lock MPTT partitions, retrying once more") + logging.error( + "Deadlock detected when attempting to lock MPTT partitions, retrying once more" + ) with self._attempt_lock_mptt(): yield else: @@ -319,7 +310,6 @@ def use_nonce(cls, nonce_value): class ScopeDefinition(models.Model): - # the identifier used to specify this scope within a certificate id = models.CharField(primary_key=True, max_length=20) @@ -369,7 +359,9 @@ def __init__(self, filter_str, params=None): :type params: dict|str """ if params is not None: - logging.warning("DEPRECATED: Constructing a filter with a template and params is deprecated. Use Filter.from_template() instead") + logging.warning( + "DEPRECATED: Constructing a filter with a template and params is deprecated. Use Filter.from_template() instead" + ) filter_str = str(Filter.from_template(filter_str, params=params)) self._filter_tuple = tuple(filter_str.split()) or ("",) @@ -493,7 +485,9 @@ def __init__(self, definition, params): # turn the scope definition filter templates into Filter objects rw_filter = Filter.from_template(definition.read_write_filter_template, params) self.read_filter = rw_filter + Filter.from_template(definition.read_filter_template, params) - self.write_filter = rw_filter + Filter.from_template(definition.write_filter_template, params) + self.write_filter = rw_filter + Filter.from_template( + definition.write_filter_template, params + ) def is_subset_of(self, other): if not self.read_filter.is_subset_of(other.read_filter): @@ -506,7 +500,4 @@ def __le__(self, other): return self.is_subset_of(other) def __eq__(self, other): - return ( - self.read_filter == other.read_filter - and self.write_filter == other.write_filter - ) + return self.read_filter == other.read_filter and self.write_filter == other.write_filter diff --git a/morango/models/core.py b/morango/models/core.py index fc97250a..06abf3ff 100644 --- a/morango/models/core.py +++ b/morango/models/core.py @@ -2,22 +2,12 @@ import json import logging import uuid -from collections import defaultdict -from collections import namedtuple +from collections import defaultdict, namedtuple from functools import reduce from django.core import exceptions -from django.db import connection -from django.db import models -from django.db import router -from django.db import transaction -from django.db.models import F -from django.db.models import Func -from django.db.models import Max -from django.db.models import Q -from django.db.models import signals -from django.db.models import TextField -from django.db.models import Value +from django.db import connection, models, router, transaction +from django.db.models import F, Func, Max, Q, TextField, Value, signals from django.db.models.deletion import Collector from django.db.models.expressions import CombinedExpression from django.db.models.fields.related import ForeignKey @@ -26,22 +16,15 @@ from django.utils.functional import cached_property from morango import proquint -from morango.constants import transfer_stages -from morango.constants import transfer_statuses +from morango.constants import transfer_stages, transfer_statuses from morango.errors import InvalidMorangoSourceId -from morango.models.certificates import Certificate -from morango.models.certificates import Filter -from morango.models.fields.uuids import sha2_uuid -from morango.models.fields.uuids import UUIDField -from morango.models.fields.uuids import UUIDModelMixin +from morango.models.certificates import Certificate, Filter +from morango.models.fields.uuids import UUIDField, UUIDModelMixin, sha2_uuid from morango.models.fsic_utils import remove_redundant_instance_counters from morango.models.manager import SyncableModelManager -from morango.models.utils import get_0_4_system_parameters -from morango.models.utils import get_0_5_mac_address -from morango.models.utils import get_0_5_system_id +from morango.models.utils import get_0_4_system_parameters, get_0_5_mac_address, get_0_5_system_id from morango.registry import syncable_models -from morango.utils import _assert -from morango.utils import SETTINGS +from morango.utils import SETTINGS, _assert logger = logging.getLogger(__name__) @@ -154,7 +137,6 @@ def get_or_create_current_instance(cls, clear_cache=False): pass with transaction.atomic(): - # check if a matching legacy instance ID is already current, and don't mess with it kwargs = get_0_4_system_parameters( database_id=DatabaseIDModel.get_or_create_current_database_id().id @@ -169,15 +151,13 @@ def get_or_create_current_instance(cls, clear_cache=False): # calculate the new ID based on system ID and mac address kwargs["system_id"] = get_0_5_system_id() kwargs["node_id"] = get_0_5_mac_address() - kwargs["id"] = sha2_uuid( - kwargs["database_id"], kwargs["system_id"], kwargs["node_id"] - ) + kwargs["id"] = sha2_uuid(kwargs["database_id"], kwargs["system_id"], kwargs["node_id"]) kwargs["current"] = True # ensure we only ever have 1 current instance ID - InstanceIDModel.objects.filter(current=True).exclude( - id=kwargs["id"] - ).update(current=False) + InstanceIDModel.objects.filter(current=True).exclude(id=kwargs["id"]).update( + current=False + ) # create the model, or get existing if one already exists with this ID instance, created = InstanceIDModel.objects.update_or_create( id=kwargs["id"], defaults=kwargs @@ -216,10 +196,18 @@ class SyncSession(models.Model): # track the certificates being used by each side for this session client_certificate = models.ForeignKey( - Certificate, blank=True, null=True, related_name="syncsessions_client", on_delete=models.CASCADE + Certificate, + blank=True, + null=True, + related_name="syncsessions_client", + on_delete=models.CASCADE, ) server_certificate = models.ForeignKey( - Certificate, blank=True, null=True, related_name="syncsessions_server", on_delete=models.CASCADE + Certificate, + blank=True, + null=True, + related_name="syncsessions_server", + on_delete=models.CASCADE, ) # track the morango profile this sync session is happening for @@ -266,9 +254,7 @@ class TransferSession(models.Model): """ id = UUIDField(primary_key=True) - filter = ( - models.TextField() - ) # partition/filter to know what subset of data is to be synced + filter = models.TextField() # partition/filter to know what subset of data is to be synced push = models.BooleanField() # is session pushing or pulling data? active = models.BooleanField(default=True) # is this transfer session still active? records_transferred = models.IntegerField( @@ -341,9 +327,7 @@ def delete_buffers(self): against the database for better performance """ with connection.cursor() as cursor: - cursor.execute( - "DELETE FROM morango_buffer WHERE transfer_session_id = %s", (self.id,) - ) + cursor.execute("DELETE FROM morango_buffer WHERE transfer_session_id = %s", (self.id,)) cursor.execute( "DELETE FROM morango_recordmaxcounterbuffer WHERE transfer_session_id = %s", (self.id,), @@ -355,9 +339,9 @@ def get_touched_record_ids_for_model(self, model): ): model = model.morango_model_name _assert(isinstance(model, str), "Model must resolve to string") - return Store.objects.filter( - model_name=model, last_transfer_session_id=self.id - ).values_list("id", flat=True) + return Store.objects.filter(model_name=model, last_transfer_session_id=self.id).values_list( + "id", flat=True + ) class DeletedModels(models.Model): @@ -418,7 +402,13 @@ def char_ids_list(self): self.annotate(id_cast=Cast("id", TextField())) # remove dashes from char uuid .annotate( - fixed_id=Func(F("id_cast"), Value("-"), Value(""), function="replace", output_field=TextField()) + fixed_id=Func( + F("id_cast"), + Value("-"), + Value(""), + function="replace", + output_field=TextField(), + ) ) # return as list .values_list("fixed_id", flat=True) @@ -441,16 +431,18 @@ class Store(AbstractStore): dirty_bit = models.BooleanField(default=False) deserialization_error = models.TextField(blank=True) - last_transfer_session_id = UUIDField( - blank=True, null=True, default=None, db_index=True - ) + last_transfer_session_id = UUIDField(blank=True, null=True, default=None, db_index=True) objects = StoreManager() class Meta: indexes = [ models.Index(fields=["partition"], name="idx_morango_store_partition"), - models.Index(fields=["profile", "model_name", "partition", "dirty_bit"], condition=models.Q(dirty_bit=True), name="idx_morango_deserialize"), + models.Index( + fields=["profile", "model_name", "partition", "dirty_bit"], + condition=models.Q(dirty_bit=True), + name="idx_morango_deserialize", + ), ] def _deserialize_store_model(self, fk_cache, defer_fks=False, sync_filter=None): # noqa: C901 @@ -479,12 +471,15 @@ def _deserialize_store_model(self, fk_cache, defer_fks=False, sync_filter=None): # Import here to avoid circular import, as the utils module # imports core models. from morango.sync.utils import mute_signals + with mute_signals(signals.post_delete): klass_model.syncing_objects.filter(id=self.id).delete() return None, deferred_fks else: # load model into memory - app_model = klass_model.deserialize(json.loads(self.serialized), sync_filter=sync_filter) + app_model = klass_model.deserialize( + json.loads(self.serialized), sync_filter=sync_filter + ) app_model._morango_source_id = self.source_id app_model._morango_partition = self.partition app_model._morango_dirty_bit = False @@ -498,7 +493,6 @@ def _deserialize_store_model(self, fk_cache, defer_fks=False, sync_filter=None): return app_model, deferred_fks except (exceptions.ValidationError, exceptions.ObjectDoesNotExist) as e: - logger.warning( "Error deserializing instance of {model} with id {id}: {error}".format( model=klass_model.__name__, id=app_model.id, error=e @@ -578,9 +572,7 @@ def __init__(self, value, field): super(ValueStartsWithField, self).__init__( Value(value, output_field=models.CharField()), "LIKE", - CombinedExpression( - F(field), "||", Value("%", output_field=models.CharField()) - ), + CombinedExpression(F(field), "||", Value("%", output_field=models.CharField())), output_field=models.BooleanField(), ) @@ -626,7 +618,7 @@ def update_fsics(cls, fsics, sync_filter, v2_format=False): updated_fsic[key] = fsics[key] # load database max counters - for (key, value) in updated_fsic.items(): + for key, value in updated_fsic.items(): for f in sync_filter: DatabaseMaxCounter.objects.update_or_create( instance_id=key, partition=f, defaults={"counter": value} @@ -679,7 +671,6 @@ def calculate_filter_specific_instance_counters( ): if v2_format: - queryset = cls.objects.all() # get the DMC records with partitions that fall under the filter prefixes @@ -688,24 +679,18 @@ def calculate_filter_specific_instance_counters( [Q(partition__startswith=prefix) for prefix in filters], ) sub_partitions = set( - queryset.filter(sub_condition) - .values_list("partition", flat=True) - .distinct() + queryset.filter(sub_condition).values_list("partition", flat=True).distinct() ) # get the DMC records with partitions that are prefixes of the filters super_partitions = set() for filt in filters: qs = ( - queryset.annotate( - filter_matches=ValueStartsWithField(filt, "partition") - ) + queryset.annotate(filter_matches=ValueStartsWithField(filt, "partition")) .filter(filter_matches=True) .exclude(partition=filt) ) - super_partitions.update( - qs.values_list("partition", flat=True).distinct() - ) + super_partitions.update(qs.values_list("partition", flat=True).distinct()) # get the instance counters for the partitions, also filtering out old unnecessary instance_ids super_fsics = cls.get_instance_counters_for_partitions( @@ -726,21 +711,16 @@ def calculate_filter_specific_instance_counters( return raw_fsic else: - queryset = cls.objects.all() per_filter_max = [] for filt in filters: # {filt} LIKE partition || '%' - qs = queryset.annotate( - filter_matches=ValueStartsWithField(filt, "partition") - ) + qs = queryset.annotate(filter_matches=ValueStartsWithField(filt, "partition")) qs = qs.filter(filter_matches=True) filt_maxes = qs.values("instance_id").annotate(maxval=Max("counter")) - per_filter_max.append( - {dmc["instance_id"]: dmc["maxval"] for dmc in filt_maxes} - ) + per_filter_max.append({dmc["instance_id"]: dmc["maxval"] for dmc in filt_maxes}) instance_id_lists = [maxes.keys() for maxes in per_filter_max] all_instance_ids = reduce(set.union, instance_id_lists, set()) @@ -754,9 +734,7 @@ def calculate_filter_specific_instance_counters( # when we're receiving, we don't want to overpromise on what we have result = { instance_id: min([d.get(instance_id, 0) for d in per_filter_max]) - for instance_id in reduce( - set.intersection, instance_id_lists, all_instance_ids - ) + for instance_id in reduce(set.intersection, instance_id_lists, all_instance_ids) } return result @@ -785,9 +763,7 @@ class RecordMaxCounterBuffer(AbstractCounter): model_uuid = UUIDField(db_index=True) -ForeignKeyReference = namedtuple( - "ForeignKeyReference", ["from_field", "from_pk", "to_pk"] -) +ForeignKeyReference = namedtuple("ForeignKeyReference", ["from_field", "from_pk", "to_pk"]) class SyncableModel(UUIDModelMixin): @@ -840,9 +816,7 @@ def save(self, update_dirty_bit_to=True, *args, **kwargs): self._morango_dirty_bit = False super(SyncableModel, self).save(*args, **kwargs) - def delete( - self, using=None, keep_parents=False, hard_delete=False, *args, **kwargs - ): + def delete(self, using=None, keep_parents=False, hard_delete=False, *args, **kwargs): using = using or router.db_for_write(self.__class__, instance=self) _assert( self._get_pk_val() is not None, @@ -885,15 +859,11 @@ def cached_clean_fields(self, fk_lookup_cache, exclude=None, sync_filter=None): :type sync_filter: Filter|None """ excluded_fields = exclude or [] - fk_fields = [ - field for field in self._meta.fields if isinstance(field, models.ForeignKey) - ] + fk_fields = [field for field in self._meta.fields if isinstance(field, models.ForeignKey)] for f in fk_fields: raw_value = getattr(self, f.attname) - key = "{id}_{db_table}".format( - db_table=f.related_model._meta.db_table, id=raw_value - ) + key = "{id}_{db_table}".format(db_table=f.related_model._meta.db_table, id=raw_value) try: fk_lookup_cache[key] excluded_fields.append(f.name) @@ -1012,9 +982,7 @@ def calculate_uuid(self): # undefined behavior, which due to dynamic nature of calculating it, this could be an unintentional bug # so we raise an error to strictly enforce this raise InvalidMorangoSourceId( - "{}.calculate_source_id() returned empty string".format( - self.__class__.__name__ - ) + "{}.calculate_source_id() returned empty string".format(self.__class__.__name__) ) namespaced_id = self.compute_namespaced_id( diff --git a/morango/models/fields/__init__.py b/morango/models/fields/__init__.py index 91de5717..ad6c8fae 100644 --- a/morango/models/fields/__init__.py +++ b/morango/models/fields/__init__.py @@ -1,5 +1,4 @@ -from morango.models.fields.crypto import PrivateKeyField -from morango.models.fields.crypto import PublicKeyField +from morango.models.fields.crypto import PrivateKeyField, PublicKeyField from morango.models.fields.uuids import UUIDField -__all__ = ['UUIDField', 'PublicKeyField', 'PrivateKeyField'] +__all__ = ["UUIDField", "PublicKeyField", "PrivateKeyField"] diff --git a/morango/models/fields/crypto.py b/morango/models/fields/crypto.py index ac680ab7..434e32a1 100644 --- a/morango/models/fields/crypto.py +++ b/morango/models/fields/crypto.py @@ -4,17 +4,16 @@ ``Key`` has methods for signing messages using a private key and verifying signed messages using a public key. ``Key`` classes are used for signing/verifying certificates that give various permissions. """ + import hashlib import re import sys import rsa as PYRSA -from django.db import models -from django.db import transaction +from django.db import models, transaction try: - from M2Crypto import RSA as M2RSA - from M2Crypto import BIO as M2BIO + from M2Crypto import BIO as M2BIO, RSA as M2RSA M2CRYPTO_EXISTS = True except ImportError: @@ -31,17 +30,17 @@ if cffi.__version_info__ < (1, 17, 1): raise ImportError - from cryptography.hazmat.backends import default_backend from cryptography import exceptions as crypto_exceptions + from cryptography.hazmat.backends import default_backend crypto_backend = default_backend() - from cryptography.hazmat.primitives.asymmetric import ( - rsa as crypto_rsa, - padding as crypto_padding, - ) from cryptography.hazmat.primitives import ( - serialization as crypto_serialization, hashes as crypto_hashes, + serialization as crypto_serialization, + ) + from cryptography.hazmat.primitives.asymmetric import ( + padding as crypto_padding, + rsa as crypto_rsa, ) # Ignore cryptography versions that do not support the 'sign' method @@ -58,8 +57,7 @@ # Otherwise raise the error again to avoid silently catching other errors raise -from base64 import encodebytes as b64encode, decodebytes as b64decode - +from base64 import decodebytes as b64decode, encodebytes as b64encode PKCS8_HEADER = "MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A" @@ -140,9 +138,7 @@ def set_private_key_string(self, private_key_string): private_key_string = self.ensure_unicode(private_key_string) - private_key_string = self._add_pem_headers( - private_key_string, "RSA PRIVATE KEY" - ) + private_key_string = self._add_pem_headers(private_key_string, "RSA PRIVATE KEY") self._set_private_key_string(private_key_string) @@ -150,11 +146,7 @@ def _remove_pem_headers(self, pem_string): if not pem_string.strip().startswith("-----"): return pem_string return "\n".join( - [ - line - for line in pem_string.split("\n") - if line and not line.startswith("---") - ] + [line for line in pem_string.split("\n") if line and not line.startswith("---")] ) def _add_pem_headers(self, pem_string, header_string): @@ -163,8 +155,7 @@ def _add_pem_headers(self, pem_string, header_string): "header_string": header_string, } return ( - "-----BEGIN %(header_string)s-----\n%(key)s\n-----END %(header_string)s-----" - % context + "-----BEGIN %(header_string)s-----\n%(key)s\n-----END %(header_string)s-----" % context ) def ensure_bytes(self, message): @@ -184,7 +175,6 @@ def __str__(self): class PythonRSAKey(BaseKey): - _public_key = None _private_key = None @@ -229,7 +219,6 @@ def _set_private_key_string(self, private_key_string): class M2CryptoKey(BaseKey): - _public_key = None _private_key = None @@ -243,9 +232,7 @@ def _sign(self, message): def _verify(self, message, signature): try: - self._public_key.verify( - hashlib.sha256(message).digest(), signature, algo="sha256" - ) + self._public_key.verify(hashlib.sha256(message).digest(), signature, algo="sha256") return True except M2RSA.RSAError: return False @@ -279,7 +266,6 @@ def _set_private_key_string(self, private_key_string): class CryptographyKey(BaseKey): - _public_key = None _private_key = None @@ -290,9 +276,7 @@ def generate_new_key(self, keysize=2048): self._public_key = self._private_key.public_key() def _sign(self, message): - return self._private_key.sign( - message, crypto_padding.PKCS1v15(), crypto_hashes.SHA256() - ) + return self._private_key.sign(message, crypto_padding.PKCS1v15(), crypto_hashes.SHA256()) def _verify(self, message, signature): try: @@ -342,11 +326,7 @@ def _set_private_key_string(self, private_key_string): # alias the most-preferred key wrapper class we have available as `Key` -Key = ( - CryptographyKey - if CRYPTOGRAPHY_EXISTS - else (M2CryptoKey if M2CRYPTO_EXISTS else PythonRSAKey) -) +Key = CryptographyKey if CRYPTOGRAPHY_EXISTS else (M2CryptoKey if M2CRYPTO_EXISTS else PythonRSAKey) class RSAKeyBaseField(models.TextField): @@ -404,6 +384,7 @@ class SharedKey(models.Model): who would like to allow certificates to be pushed to the server must also enable ``ALLOW_CERTIFICATE_PUSHING``. Clients generate a ``Certificate`` object and set the ``public_key`` field to the shared public key of the server. """ + public_key = PublicKeyField() private_key = PrivateKeyField() current = models.BooleanField(default=True) @@ -418,14 +399,10 @@ def get_or_create_shared_key(cls, force_new=False): with transaction.atomic(): SharedKey.objects.filter(current=True).update(current=False) key = Key() - return SharedKey.objects.create( - public_key=key, private_key=key, current=True - ) + return SharedKey.objects.create(public_key=key, private_key=key, current=True) # create a new shared key if one doesn't exist try: return SharedKey.objects.get(current=True) except SharedKey.DoesNotExist: key = Key() - return SharedKey.objects.create( - public_key=key, private_key=key, current=True - ) + return SharedKey.objects.create(public_key=key, private_key=key, current=True) diff --git a/morango/models/fields/uuids.py b/morango/models/fields/uuids.py index 0f1f23ee..0e84096c 100644 --- a/morango/models/fields/uuids.py +++ b/morango/models/fields/uuids.py @@ -10,36 +10,57 @@ def sha2_uuid(*args): return hashlib.sha256("::".join(args).encode("utf-8")).hexdigest()[:32] -class UUIDField(models.UUIDField): +class UUIDField(models.CharField): """ Adaptation of Django's UUIDField, but with 32-char hex representation as Python representation rather than a UUID instance. """ + def __init__(self, *args, **kwargs): + kwargs["max_length"] = 32 + super(UUIDField, self).__init__(*args, **kwargs) + + def prepare_value(self, value): + if isinstance(value, uuid.UUID): + return value.hex + return value + + def deconstruct(self): + name, path, args, kwargs = super(UUIDField, self).deconstruct() + del kwargs["max_length"] + return name, path, args, kwargs + + def get_internal_type(self): + return "UUIDField" + def get_db_prep_value(self, value, connection, prepared=False): if value is None: return None if not isinstance(value, uuid.UUID): - value = super(UUIDField, self).to_python(value) - - if connection.features.has_native_uuid_field: - return value + try: + value = uuid.UUID(value) + except AttributeError: + raise TypeError(self.error_messages["invalid"] % {"value": value}) return value.hex def from_db_value(self, value, expression, connection): return self.to_python(value) def to_python(self, value): - value = super(UUIDField, self).to_python(value) - return value.hex if isinstance(value, uuid.UUID) else value + if isinstance(value, uuid.UUID): + return value.hex + return value def get_default(self): - default = super(UUIDField, self).get_default() - if isinstance(default, uuid.UUID): - return default.hex - return default - - def value_from_object(self, obj): - return self.to_python(super(UUIDField, self).value_from_object(obj)) + if self.has_default(): + if callable(self.default): + default = self.default() + if isinstance(default, uuid.UUID): + return default.hex + return default + if isinstance(self.default, uuid.UUID): + return self.default.hex + return self.default + return None class UUIDModelMixin(models.Model): diff --git a/morango/models/fsic_utils.py b/morango/models/fsic_utils.py index f6729b80..4455e8ec 100644 --- a/morango/models/fsic_utils.py +++ b/morango/models/fsic_utils.py @@ -137,9 +137,7 @@ def calculate_directional_fsic_diff_v2(fsic1, fsic2): :param fsic2: dict containing FSIC v2 in expanded format, for the receiving device :return ``dict`` in expanded FSIC v2 format to be used in queueing the correct records to the buffer """ - prefixes = _build_prefix_mapper( - list(fsic1.keys()) + list(fsic2.keys()), include_self=True - ) + prefixes = _build_prefix_mapper(list(fsic1.keys()) + list(fsic2.keys()), include_self=True) result = defaultdict(dict) @@ -148,9 +146,7 @@ def calculate_directional_fsic_diff_v2(fsic1, fsic2): # check for counters in the sending FSIC that are higher than the receiving FSIC for inst, sending_counter in insts.items(): # get the maximum counter in the receiving FSIC for the same instance - receiving_counter = max( - fsic2.get(prefix, {}).get(inst, 0) for prefix in prefixes[part] - ) + receiving_counter = max(fsic2.get(prefix, {}).get(inst, 0) for prefix in prefixes[part]) if receiving_counter < sending_counter: result[part][inst] = receiving_counter diff --git a/morango/models/utils.py b/morango/models/utils.py index a221f6d7..33b5e38b 100644 --- a/morango/models/utils.py +++ b/morango/models/utils.py @@ -40,9 +40,7 @@ def get_0_4_system_parameters(database_id): mac = uuid.getnode() if (mac >> 40) % 2 == 0: # 8th bit (of 48 bits, from left) is 1 if MAC is fake hashable_identifier = "{}:{}".format(params["database_id"], mac) - params["node_id"] = hashlib.sha1( - hashable_identifier.encode("utf-8") - ).hexdigest()[:20] + params["node_id"] = hashlib.sha1(hashable_identifier.encode("utf-8")).hexdigest()[:20] else: params["node_id"] = "" @@ -75,9 +73,7 @@ def _calculate_0_4_uuid(parameters): def _query_wmic(namespace, key): try: result = ( - subprocess.check_output("wmic {} get {}".format(namespace, key)) - .decode() - .split()[-1] + subprocess.check_output("wmic {} get {}".format(namespace, key)).decode().split()[-1] ) if "-" in result: @@ -132,7 +128,6 @@ def get_0_5_system_id(): # Windows elif sys.platform == "win32": - # try to get the system serial number, if available system_id = _query_wmic("csproduct", "UUID") # if UUID consists only of "F" digits, it's not usable diff --git a/morango/proquint.py b/morango/proquint.py index 0280f43b..2550f8a7 100644 --- a/morango/proquint.py +++ b/morango/proquint.py @@ -3,6 +3,7 @@ The simplest ways to use this module are the :func:`humanize` and :func:`uuid` functions. For tighter control over the output, see :class:`HumanHasher`. """ + import uuid # Copyright (c) 2014 SUNET. All rights reserved. diff --git a/morango/registry.py b/morango/registry.py index 0196d277..25a1f924 100644 --- a/morango/registry.py +++ b/morango/registry.py @@ -2,6 +2,7 @@ `SyncableModelRegistry` holds all syncable models for a project, on a per profile basis. This class is registered at app load time for morango in `apps.py`. """ + import inspect import sys from collections import OrderedDict @@ -11,17 +12,16 @@ from django.db.models.fields.related import ForeignKey from morango.constants import transfer_stages -from morango.errors import InvalidMorangoModelConfiguration -from morango.errors import ModelRegistryNotReady -from morango.errors import UnsupportedFieldType -from morango.utils import do_import -from morango.utils import SETTINGS +from morango.errors import ( + InvalidMorangoModelConfiguration, + ModelRegistryNotReady, + UnsupportedFieldType, +) +from morango.utils import SETTINGS, do_import def _get_foreign_key_classes(m): - return set( - [field.related_model for field in m._meta.fields if isinstance(field, ForeignKey)] - ) + return set([field.related_model for field in m._meta.fields if isinstance(field, ForeignKey)]) def _multiple_self_ref_fk_check(class_model): @@ -40,18 +40,15 @@ def _multiple_self_ref_fk_check(class_model): def _check_manager(name, objects): from morango.models.manager import SyncableModelManager from morango.models.query import SyncableModelQuerySet + # syncable model checks if not isinstance(objects, SyncableModelManager): raise InvalidMorangoModelConfiguration( - "Manager for {} must inherit from SyncableModelManager.".format( - name - ) + "Manager for {} must inherit from SyncableModelManager.".format(name) ) if not isinstance(objects.none(), SyncableModelQuerySet): raise InvalidMorangoModelConfiguration( - "Queryset for {} model must inherit from SyncableModelQuerySet.".format( - name - ) + "Queryset for {} model must inherit from SyncableModelQuerySet.".format(name) ) @@ -66,9 +63,7 @@ def __init__(self): def check_models_ready(self, profile): """Raise an exception if all models haven't been imported yet.""" if not self.models_ready.get(profile): - raise ModelRegistryNotReady( - "Models for profile {} aren't loaded yet.".format(profile) - ) + raise ModelRegistryNotReady("Models for profile {} aren't loaded yet.".format(profile)) def get_model(self, profile, model_name): """ @@ -102,9 +97,7 @@ def _insert_model_in_dependency_order(self, model, profile): # add any more specified dependencies if hasattr(model, "morango_model_dependencies"): - foreign_key_classes = foreign_key_classes | set( - model.morango_model_dependencies - ) + foreign_key_classes = foreign_key_classes | set(model.morango_model_dependencies) # Find all the existing models that this new model refers to. class_indices = [ @@ -125,6 +118,7 @@ def populate(self): # noqa: C901 return import django.apps + from morango.models.core import SyncableModel model_list = [] @@ -146,9 +140,7 @@ def populate(self): # noqa: C901 ) if not hasattr(model, "morango_model_name"): raise InvalidMorangoModelConfiguration( - "{} model must define a morango_model_name attribute".format( - name - ) + "{} model must define a morango_model_name attribute".format(name) ) if not hasattr(model, "morango_profile"): raise InvalidMorangoModelConfiguration( diff --git a/morango/sync/backends/base.py b/morango/sync/backends/base.py index 5e452be5..8c37db0b 100644 --- a/morango/sync/backends/base.py +++ b/morango/sync/backends/base.py @@ -1,9 +1,6 @@ from contextlib import contextmanager -from morango.models.core import Buffer -from morango.models.core import RecordMaxCounter -from morango.models.core import RecordMaxCounterBuffer -from morango.models.core import Store +from morango.models.core import Buffer, RecordMaxCounter, RecordMaxCounterBuffer, Store class BaseSQLWrapper(object): @@ -37,16 +34,14 @@ def _bulk_full_record_upsert(self, cursor, table_name, fields, db_values): raise NotImplementedError("Subclass must implement this method.") def _bulk_insert(self, cursor, table_name, fields, db_values): - placeholder_str = ", ".join( - self._create_placeholder_list(fields, db_values) - ).replace("'", "") + placeholder_str = ", ".join(self._create_placeholder_list(fields, db_values)).replace( + "'", "" + ) fields_str = str(tuple(str(f.attname) for f in fields)).replace("'", "") insert = """ INSERT INTO {table_name} {fields} VALUES {placeholder_str} - """.format( - table_name=table_name, fields=fields_str, placeholder_str=placeholder_str - ) + """.format(table_name=table_name, fields=fields_str, placeholder_str=placeholder_str) cursor.execute(insert, db_values) def _bulk_update(self, cursor, table_name, fields, db_values): @@ -102,9 +97,7 @@ def _dequeuing_merge_conflict_rmcb(self, cursor, transfersession_id): def _dequeuing_merge_conflict_buffer(self, cursor, current_id, transfersession_id): raise NotImplementedError("Subclass must implement this method.") - def _dequeuing_update_rmcs_last_saved_by( - self, cursor, current_id, transfersession_id - ): + def _dequeuing_update_rmcs_last_saved_by(self, cursor, current_id, transfersession_id): raise NotImplementedError("Subclass must implement this method.") def _dequeuing_delete_mc_buffer(self, cursor, transfersession_id): @@ -187,9 +180,7 @@ def _create_temporary_table(self, cursor, name, field_sqls, fields_params): :param field_sqls: A list of SQL strings representing the fields :param fields_params: A list of SQL parameters if necessary for the fields SQL """ - sql = self.create_temporary_table_template.format( - name=name, fields=", ".join(field_sqls) - ) + sql = self.create_temporary_table_template.format(name=name, fields=", ".join(field_sqls)) cursor.execute(sql, fields_params) def _lock_all_partitions(self, shared=False): diff --git a/morango/sync/backends/utils.py b/morango/sync/backends/utils.py index 856e71df..527dc542 100644 --- a/morango/sync/backends/utils.py +++ b/morango/sync/backends/utils.py @@ -31,15 +31,13 @@ def calculate_max_sqlite_variables(): MAX_VARIABLE_NUMBER = 999 # check that target compilation option is specified, before we start looping through - is_defined = list( - conn.execute("SELECT sqlite_compileoption_used('MAX_VARIABLE_NUMBER');") - )[0][0] + is_defined = list(conn.execute("SELECT sqlite_compileoption_used('MAX_VARIABLE_NUMBER');"))[0][ + 0 + ] if is_defined: for i in range(500): - option_str = list(conn.execute("SELECT sqlite_compileoption_get(?);", [i]))[ - 0 - ][0] + option_str = list(conn.execute("SELECT sqlite_compileoption_get(?);", [i]))[0][0] if option_str is None: # we've hit the end of the compilation options, so we can stop break @@ -112,9 +110,7 @@ def create(self): schema_editor = self.connection.schema_editor() for field in self.fields: # generates the SQL expression for the table column - field_sql, field_params = schema_editor.column_sql( - self, field, include_default=True - ) + field_sql, field_params = schema_editor.column_sql(self, field, include_default=True) field_sql_name = self.connection.ops.quote_name(field.column) fields.append("{name} {sql}".format(name=field_sql_name, sql=field_sql)) params.extend(field_params) diff --git a/morango/sync/context.py b/morango/sync/context.py index 794d68b8..87a3a410 100644 --- a/morango/sync/context.py +++ b/morango/sync/context.py @@ -1,11 +1,8 @@ -from morango.constants import transfer_stages -from morango.constants import transfer_statuses +from morango.constants import transfer_stages, transfer_statuses from morango.errors import MorangoContextUpdateError from morango.models.certificates import Filter -from morango.models.core import SyncSession -from morango.models.core import TransferSession -from morango.utils import CAPABILITIES -from morango.utils import parse_capabilities_from_server_request +from morango.models.core import SyncSession, TransferSession +from morango.utils import CAPABILITIES, parse_capabilities_from_server_request class SessionContext(object): @@ -90,7 +87,11 @@ def update( :type capabilities: str[]|None :type error: BaseException|None """ - if transfer_session and self.transfer_session and transfer_session.id != self.transfer_session.id: + if ( + transfer_session + and self.transfer_session + and transfer_session.id != self.transfer_session.id + ): raise MorangoContextUpdateError("Transfer session already exists") elif ( transfer_session @@ -101,8 +102,12 @@ def update( if sync_filter and self.filter and sync_filter != self.filter: if not self.filter.is_subset_of(sync_filter): - raise MorangoContextUpdateError("The existing filter must be a subset of the new filter") - if transfer_stages.stage(self.stage) > transfer_stages.stage(transfer_stages.INITIALIZING): + raise MorangoContextUpdateError( + "The existing filter must be a subset of the new filter" + ) + if transfer_stages.stage(self.stage) > transfer_stages.stage( + transfer_stages.INITIALIZING + ): raise MorangoContextUpdateError("Cannot update filter after initializing stage") if is_push is not None and self.is_push is not None: @@ -160,9 +165,7 @@ def __getstate__(self): """Return dict of simplified data for serialization""" return dict( sync_session_id=self.sync_session.id if self.sync_session else None, - transfer_session_id=( - self.transfer_session.id if self.transfer_session else None - ), + transfer_session_id=(self.transfer_session.id if self.transfer_session else None), filter=str(self.filter), is_push=self.is_push, stage=self.stage, @@ -465,9 +468,7 @@ def update_state(self, stage=None, stage_status=None): return # advance the composite's stage when we move forward only - if stage is not None and transfer_stages.stage(stage) > transfer_stages.stage( - self._stage - ): + if stage is not None and transfer_stages.stage(stage) > transfer_stages.stage(self._stage): self._stage = stage # when finishing a stage without an error, we'll increment the counter by one such that @@ -477,10 +478,7 @@ def update_state(self, stage=None, stage_status=None): # when we've completed a loop through all contexts (modulus is zero), we want to bring # all the contexts' states up to date - if ( - self._counter % len(self.children) == 0 - or stage_status == transfer_statuses.ERRORED - ): + if self._counter % len(self.children) == 0 or stage_status == transfer_statuses.ERRORED: for context in self.children: context.update_state(stage=stage, stage_status=stage_status) if stage_status is not None: diff --git a/morango/sync/controller.py b/morango/sync/controller.py index fa64ede9..c076b284 100644 --- a/morango/sync/controller.py +++ b/morango/sync/controller.py @@ -2,11 +2,9 @@ import math from time import sleep -from morango.constants import transfer_stages -from morango.constants import transfer_statuses +from morango.constants import transfer_stages, transfer_statuses from morango.registry import session_middleware -from morango.sync.operations import _deserialize_from_store -from morango.sync.operations import OperationLogger +from morango.sync.operations import OperationLogger, _deserialize_from_store from morango.sync.stream.serialize import serialize_into_store from morango.sync.utils import SyncSignalGroup from morango.utils import _assert @@ -33,9 +31,7 @@ def deserialize_from_store(self, skip_erroring=False, sync_filter=None): with OperationLogger("Deserializing records", "Deserialization complete"): # we first serialize to avoid deserialization merge conflicts serialize_into_store(self.profile, sync_filter=sync_filter) - _deserialize_from_store( - self.profile, filter=sync_filter, skip_erroring=skip_erroring - ) + _deserialize_from_store(self.profile, filter=sync_filter, skip_erroring=skip_erroring) def create_network_connection(self, base_url, **kwargs): from morango.sync.syncsession import NetworkSyncConnection @@ -175,9 +171,7 @@ def proceed_to(self, target_stage, context=None): # should always be a non-False status return result - def proceed_to_and_wait_for( - self, target_stage, context=None, max_interval=None, callback=None - ): + def proceed_to_and_wait_for(self, target_stage, context=None, max_interval=None, callback=None): """ Same as `proceed_to` but waits for a finished status to be returned by sleeping between calls to `proceed_to` if status is not complete diff --git a/morango/sync/db.py b/morango/sync/db.py index 5bf56ed5..6ee04674 100644 --- a/morango/sync/db.py +++ b/morango/sync/db.py @@ -1,13 +1,11 @@ import logging from contextlib import contextmanager -from django.db import connection -from django.db import transaction +from django.db import connection, transaction from morango.sync.backends.utils import load_backend from morango.sync.utils import lock_partitions - logger = logging.getLogger(__name__) DBBackend = load_backend(connection) diff --git a/morango/sync/session.py b/morango/sync/session.py index bbe03161..c8c748a1 100644 --- a/morango/sync/session.py +++ b/morango/sync/session.py @@ -1,14 +1,12 @@ import logging from requests import exceptions -from morango import __version__ +from requests.packages.urllib3.util.url import parse_url from requests.sessions import Session from requests.utils import super_len -from requests.packages.urllib3.util.url import parse_url - -from morango.utils import serialize_capabilities_to_client_request -from morango.utils import SETTINGS +from morango import __version__ +from morango.utils import SETTINGS, serialize_capabilities_to_client_request logger = logging.getLogger(__name__) @@ -24,9 +22,7 @@ def _headers_content_length(headers): def _length_of_headers(headers): - return super_len( - "\n".join(["{}: {}".format(key, value) for key, value in headers.items()]) - ) + return super_len("\n".join(["{}: {}".format(key, value) for key, value in headers.items()])) class SessionWrapper(Session): @@ -43,7 +39,9 @@ def __init__(self): if SETTINGS.CUSTOM_INSTANCE_INFO is not None: instances = list(SETTINGS.CUSTOM_INSTANCE_INFO) if instances: - user_agent_header += " " + "{}/{}".format(instances[0], SETTINGS.CUSTOM_INSTANCE_INFO.get(instances[0])) + user_agent_header += " " + "{}/{}".format( + instances[0], SETTINGS.CUSTOM_INSTANCE_INFO.get(instances[0]) + ) self.headers["User-Agent"] = "{} {}".format(user_agent_header, self.headers["User-Agent"]) def request(self, method, url, **kwargs): @@ -71,9 +69,7 @@ def request(self, method, url, **kwargs): response = req_err.response response_content = response.content if response else "(no response)" - logger.error( - "{} Reason: {}".format(req_err.__class__.__name__, response_content) - ) + logger.error("{} Reason: {}".format(req_err.__class__.__name__, response_content)) raise req_err def prepare_request(self, request): diff --git a/morango/sync/stream/core.py b/morango/sync/stream/core.py index 5892829d..aed94a2a 100644 --- a/morango/sync/stream/core.py +++ b/morango/sync/stream/core.py @@ -4,14 +4,9 @@ Provides a modular source > transform > sink pattern where models are streamed one-by-one through a pipeline of connected modules, reducing memory overhead. """ + import abc -from typing import Any -from typing import Generic -from typing import Iterable -from typing import Iterator -from typing import List -from typing import Optional -from typing import TypeVar +from typing import Any, Generic, Iterable, Iterator, List, Optional, TypeVar T = TypeVar("T") @@ -93,9 +88,7 @@ class Pipeline(PipelineModule): each module feeds into the next. """ - def __init__( - self, source: Source, modules: Optional[List[OperatorModule]] = None - ) -> None: + def __init__(self, source: Source, modules: Optional[List[OperatorModule]] = None) -> None: self._source = source self._modules = list(modules) if modules else [] diff --git a/morango/sync/stream/serialize.py b/morango/sync/stream/serialize.py index 5488e8eb..935a831f 100644 --- a/morango/sync/stream/serialize.py +++ b/morango/sync/stream/serialize.py @@ -1,29 +1,22 @@ import json import logging -from typing import Generator -from typing import Iterable -from typing import Iterator -from typing import List -from typing import Optional -from typing import Type +from typing import Generator, Iterable, Iterator, List, Optional, Type from django.core.serializers.json import DjangoJSONEncoder from django.db.models import Q from morango.models.certificates import Filter -from morango.models.core import DatabaseMaxCounter -from morango.models.core import DeletedModels -from morango.models.core import HardDeletedModels -from morango.models.core import InstanceIDModel -from morango.models.core import RecordMaxCounter -from morango.models.core import Store -from morango.models.core import SyncableModel +from morango.models.core import ( + DatabaseMaxCounter, + DeletedModels, + HardDeletedModels, + InstanceIDModel, + RecordMaxCounter, + Store, + SyncableModel, +) from morango.registry import syncable_models -from morango.sync.stream.core import Buffer -from morango.sync.stream.core import Sink -from morango.sync.stream.core import Source -from morango.sync.stream.core import Transform -from morango.sync.stream.core import Unbuffer +from morango.sync.stream.core import Buffer, Sink, Source, Transform, Unbuffer from morango.utils import self_referential_fk logger = logging.getLogger(__name__) @@ -276,9 +269,7 @@ def consume(self, tasks: List[SerializeTask]): # noqa: C901 ) if stores_to_create: - created_stores = Store.objects.bulk_create( - stores_to_create, ignore_conflicts=True - ) + created_stores = Store.objects.bulk_create(stores_to_create, ignore_conflicts=True) for created_store in created_stores: # if bulk_create has not marked it as saving been added, then it must have been # a conflict, so we'll add it to the update list @@ -315,9 +306,7 @@ def consume(self, tasks: List[SerializeTask]): # noqa: C901 app_model_ids = [task.obj.id for task in tasks] app_model = tasks[0].model - app_model.syncing_objects.filter(id__in=app_model_ids).update( - update_dirty_bit_to=False - ) + app_model.syncing_objects.filter(id__in=app_model_ids).update(update_dirty_bit_to=False) def finalize(self): self._handle_deleted() @@ -361,9 +350,9 @@ def _handle_deleted(self): DeletedModels.objects.filter(profile=self.profile).delete() def _handle_hard_deleted(self): - hard_deleted_ids = HardDeletedModels.objects.filter( - profile=self.profile - ).values_list("id", flat=True) + hard_deleted_ids = HardDeletedModels.objects.filter(profile=self.profile).values_list( + "id", flat=True + ) hard_deleted_store_records = Store.objects.filter(id__in=hard_deleted_ids) hard_deleted_store_records.update( diff --git a/morango/sync/syncsession.py b/morango/sync/syncsession.py index b3f61603..c74d73d0 100644 --- a/morango/sync/syncsession.py +++ b/morango/sync/syncsession.py @@ -1,48 +1,39 @@ """ The main module to be used for initiating the synchronization of data between morango instances. """ + import json import logging import os import socket import uuid from io import BytesIO -from urllib.parse import urljoin -from urllib.parse import urlparse +from urllib.parse import urljoin, urlparse +from django.db import connection, transaction from django.utils import timezone from requests.adapters import HTTPAdapter from requests.exceptions import HTTPError from requests.packages.urllib3.util.retry import Retry -from django.db import transaction, connection -from .session import SessionWrapper -from morango.api.serializers import CertificateSerializer -from morango.api.serializers import InstanceIDSerializer -from morango.constants import api_urls -from morango.constants import transfer_stages -from morango.constants import transfer_statuses -from morango.constants.capabilities import ALLOW_CERTIFICATE_PUSHING -from morango.constants.capabilities import GZIP_BUFFER_POST -from morango.errors import CertificateSignatureInvalid -from morango.errors import MorangoError -from morango.errors import MorangoResumeSyncError -from morango.errors import MorangoServerDoesNotAllowNewCertPush -from morango.models.certificates import Certificate -from morango.models.certificates import Filter -from morango.models.certificates import Key -from morango.models.core import InstanceIDModel -from morango.models.core import SyncSession +from morango.api.serializers import CertificateSerializer, InstanceIDSerializer +from morango.constants import api_urls, transfer_stages, transfer_statuses +from morango.constants.capabilities import ALLOW_CERTIFICATE_PUSHING, GZIP_BUFFER_POST +from morango.errors import ( + CertificateSignatureInvalid, + MorangoError, + MorangoResumeSyncError, + MorangoServerDoesNotAllowNewCertPush, +) +from morango.models.certificates import Certificate, Filter, Key +from morango.models.core import InstanceIDModel, SyncSession from morango.sync.backends.utils import load_backend -from morango.sync.context import CompositeSessionContext -from morango.sync.context import LocalSessionContext -from morango.sync.context import NetworkSessionContext +from morango.sync.context import CompositeSessionContext, LocalSessionContext, NetworkSessionContext from morango.sync.controller import SessionController -from morango.sync.utils import SyncSignal -from morango.sync.utils import SyncSignalGroup -from morango.utils import CAPABILITIES -from morango.utils import pid_exists -from morango.sync.utils import lock_partitions +from morango.sync.utils import SyncSignal, SyncSignalGroup, lock_partitions +from morango.utils import CAPABILITIES, pid_exists + +from .session import SessionWrapper if GZIP_BUFFER_POST in CAPABILITIES: from gzip import GzipFile @@ -52,6 +43,7 @@ DBBackend = load_backend(connection) + def _join_with_logical_operator(lst, operator): op = ") {operator} (".format(operator=operator) return "(({items}))".format(items=op.join(lst)) @@ -79,9 +71,7 @@ def _get_client_ip_for_server(server_host, server_port): # borrowed from https://github.com/django/django/blob/1.11.20/django/utils/text.py#L295 def compress_string(s, compresslevel=9): zbuf = BytesIO() - with GzipFile( - mode="wb", compresslevel=compresslevel, fileobj=zbuf, mtime=0 - ) as zfile: + with GzipFile(mode="wb", compresslevel=compresslevel, fileobj=zbuf, mtime=0) as zfile: zfile.write(s) return zbuf.getvalue() @@ -138,9 +128,7 @@ def __init__( self.session.mount("http://", adapter) self.session.mount("https://", adapter) # get morango information about server - self.server_info = self.session.get( - urljoin(self.base_url, api_urls.INFO) - ).json() + self.server_info = self.session.get(urljoin(self.base_url, api_urls.INFO)).json() self.capabilities = self.server_info.get("capabilities", []) self.chunk_size = chunk_size @@ -202,15 +190,11 @@ def create_sync_session(self, client_cert, server_cert, chunk_size=None): "client_certificate_id": client_cert.id, "profile": client_cert.profile, "certificate_chain": json.dumps( - CertificateSerializer( - client_cert.get_ancestors(include_self=True), many=True - ).data + CertificateSerializer(client_cert.get_ancestors(include_self=True), many=True).data ), "connection_path": self.base_url, "instance": json.dumps( - InstanceIDSerializer( - InstanceIDModel.get_or_create_current_instance()[0] - ).data + InstanceIDSerializer(InstanceIDModel.get_or_create_current_instance()[0]).data ), "nonce": nonce, "client_ip": _get_client_ip_for_server(hostname, port), @@ -251,11 +235,7 @@ def create_sync_session(self, client_cert, server_cert, chunk_size=None): "client_ip": data["client_ip"], "server_ip": data["server_ip"], "client_instance_id": client_instance.id, - "client_instance_json": json.dumps( - InstanceIDSerializer( - client_instance - ).data - ), + "client_instance_json": json.dumps(InstanceIDSerializer(client_instance).data), "server_instance_id": server_instance_id, "server_instance_json": session_resp.json().get("server_instance") or "{}", "process_id": os.getpid(), @@ -282,9 +262,7 @@ def resume_sync_session(self, sync_session_id, chunk_size=None, ignore_existing_ try: sync_session = SyncSession.objects.get(pk=sync_session_id, active=True) except SyncSession.DoesNotExist: - raise MorangoResumeSyncError( - "Session for ID '{}' not found".format(sync_session_id) - ) + raise MorangoResumeSyncError("Session for ID '{}' not found".format(sync_session_id)) # check that process of existing session isn't still running if ( @@ -329,16 +307,12 @@ def get_remote_certificates(self, primary_partition, scope_def_id=None): # inflate remote certs into a list of unsaved models for cert in remote_certs_resp.json(): - remote_certs.append( - Certificate.deserialize(cert["serialized"], cert["signature"]) - ) + remote_certs.append(Certificate.deserialize(cert["serialized"], cert["signature"])) # filter certs by scope definition id, if provided if scope_def_id: remote_certs = [ - cert - for cert in remote_certs - if cert.scope_definition_id == scope_def_id + cert for cert in remote_certs if cert.scope_definition_id == scope_def_id ] return remote_certs @@ -362,9 +336,7 @@ def certificate_signing_request( # check again, now that we have a lock if not Certificate.objects.filter(id=parent_cert.id).exists(): # upon receiving cert chain from server, we attempt to save the chain into our records - Certificate.save_certificate_chain( - cert_chain, expected_last_id=parent_cert.id - ) + Certificate.save_certificate_chain(cert_chain, expected_last_id=parent_cert.id) csr_key = Key() # build up data for csr @@ -380,9 +352,7 @@ def certificate_signing_request( csr_data = csr_resp.json() # verify cert returned from server, and proceed to save into our records - csr_cert = Certificate.deserialize( - csr_data["serialized"], csr_data["signature"] - ) + csr_cert = Certificate.deserialize(csr_data["serialized"], csr_data["signature"]) csr_cert.private_key = csr_key csr_cert.check_certificate() csr_cert.save() @@ -392,9 +362,7 @@ def push_signed_client_certificate_chain( self, local_parent_cert, scope_definition_id, scope_params ): if ALLOW_CERTIFICATE_PUSHING not in self.capabilities: - raise MorangoServerDoesNotAllowNewCertPush( - "Server does not allow certificate pushing" - ) + raise MorangoServerDoesNotAllowNewCertPush("Server does not allow certificate pushing") # grab shared public key of server publickey_response = self._get_public_key() @@ -409,12 +377,8 @@ def push_signed_client_certificate_chain( scope_definition_id=scope_definition_id, scope_version=local_parent_cert.scope_version, scope_params=json.dumps(scope_params), - public_key=Key( - public_key_string=publickey_response.json()[0]["public_key"] - ), - salt=nonce_response.json()[ - "id" - ], # for pushing signed certs, we use nonce as salt + public_key=Key(public_key_string=publickey_response.json()[0]["public_key"]), + salt=nonce_response.json()["id"], # for pushing signed certs, we use nonce as salt ) # add ID and signature to the certificate @@ -422,9 +386,7 @@ def push_signed_client_certificate_chain( certificate.parent.sign_certificate(certificate) # serialize the chain for sending to server - certificate_chain = list(local_parent_cert.get_ancestors(include_self=True)) + [ - certificate - ] + certificate_chain = list(local_parent_cert.get_ancestors(include_self=True)) + [certificate] data = json.dumps(CertificateSerializer(certificate_chain, many=True).data) # client sends signed certificate chain to server @@ -446,9 +408,7 @@ def _get_certificate_chain(self, params): def _certificate_signing(self, data, userargs, password): # convert user arguments into query str for passing to auth layer if isinstance(userargs, dict): - userargs = "&".join( - ["{}={}".format(key, val) for (key, val) in userargs.items()] - ) + userargs = "&".join(["{}={}".format(key, val) for (key, val) in userargs.items()]) return self.session.post( self.urlresolve(api_urls.CERTIFICATE), json=data, auth=(userargs, password) ) @@ -460,9 +420,7 @@ def _create_sync_session(self, data): return self.session.post(self.urlresolve(api_urls.SYNCSESSION), json=data) def _get_sync_session(self, sync_session): - return self.session.get( - self.urlresolve(api_urls.SYNCSESSION, lookup=sync_session.id) - ) + return self.session.get(self.urlresolve(api_urls.SYNCSESSION, lookup=sync_session.id)) def _create_transfer_session(self, data): return self.session.post(self.urlresolve(api_urls.TRANSFERSESSION), json=data) @@ -484,9 +442,7 @@ def _close_transfer_session(self, transfer_session): ) def _close_sync_session(self, sync_session): - return self.session.delete( - self.urlresolve(api_urls.SYNCSESSION, lookup=sync_session.id) - ) + return self.session.delete(self.urlresolve(api_urls.SYNCSESSION, lookup=sync_session.id)) def _push_record_chunk(self, data): # gzip the data if both client and server have gzipping capabilities @@ -651,9 +607,7 @@ def initialize(self, sync_filter): error_msg="Failed to initialize transfer session", ) - self.signals.session.started.fire( - transfer_session=self.current_transfer_session - ) + self.signals.session.started.fire(transfer_session=self.current_transfer_session) # backwards compatibility for the queuing signal as it included both serialization # and queuing originally @@ -673,15 +627,11 @@ def run(self): ) def finalize(self): - with self.signals.dequeuing.send( - transfer_session=self.current_transfer_session - ): + with self.signals.dequeuing.send(transfer_session=self.current_transfer_session): self.proceed_to_and_wait_for(transfer_stages.DESERIALIZING) self.proceed_to_and_wait_for(transfer_stages.CLEANUP) - self.signals.session.completed.fire( - transfer_session=self.current_transfer_session - ) + self.signals.session.completed.fire(transfer_session=self.current_transfer_session) class PushClient(TransferClient): diff --git a/morango/sync/utils.py b/morango/sync/utils.py index a025b55e..05aa1232 100644 --- a/morango/sync/utils.py +++ b/morango/sync/utils.py @@ -5,12 +5,9 @@ from django.db import transaction from rest_framework.exceptions import ValidationError -from morango.models.core import Buffer -from morango.models.core import RecordMaxCounterBuffer -from morango.models.core import SyncableModel +from morango.models.core import Buffer, RecordMaxCounterBuffer, SyncableModel from morango.registry import syncable_models - logger = logging.getLogger(__name__) @@ -83,9 +80,7 @@ def validate_and_create_buffer_data( # noqa: C901 except KeyError: Model = SyncableModel - partition = record["partition"].replace( - record["model_uuid"], Model.ID_PLACEHOLDER - ) + partition = record["partition"].replace(record["model_uuid"], Model.ID_PLACEHOLDER) expected_model_uuid = Model.compute_namespaced_id( partition, record["source_id"], record["model_name"] ) diff --git a/morango/urls.py b/morango/urls.py index 01799d66..497befeb 100644 --- a/morango/urls.py +++ b/morango/urls.py @@ -1,4 +1,3 @@ -from django.urls import include -from django.urls import path +from django.urls import include, path urlpatterns = [path("api/morango/v1/", include("morango.api.urls"))] diff --git a/morango/utils.py b/morango/utils.py index e4589fe1..0143f083 100644 --- a/morango/utils.py +++ b/morango/utils.py @@ -4,10 +4,12 @@ from django.conf import settings from morango.constants import settings as default_settings -from morango.constants.capabilities import ALLOW_CERTIFICATE_PUSHING -from morango.constants.capabilities import ASYNC_OPERATIONS -from morango.constants.capabilities import FSIC_V2_FORMAT -from morango.constants.capabilities import GZIP_BUFFER_POST +from morango.constants.capabilities import ( + ALLOW_CERTIFICATE_PUSHING, + ASYNC_OPERATIONS, + FSIC_V2_FORMAT, + GZIP_BUFFER_POST, +) def do_import(import_string): @@ -64,9 +66,7 @@ def get_capabilities(): CAPABILITIES = get_capabilities() CAPABILITIES_CLIENT_HEADER = "X-Morango-Capabilities" -CAPABILITIES_SERVER_HEADER = "HTTP_{}".format( - CAPABILITIES_CLIENT_HEADER.upper().replace("-", "_") -) +CAPABILITIES_SERVER_HEADER = "HTTP_{}".format(CAPABILITIES_CLIENT_HEADER.upper().replace("-", "_")) def serialize_capabilities_to_client_request(request): diff --git a/pyproject.toml b/pyproject.toml index 2e5db43c..beb3b70f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -97,26 +97,36 @@ morango = ["*.*"] version_file = "morango/_version.py" git_describe_command = "git describe --dirty --tags --long --match 'v*' --first-parent" -[tool.flake8] -max-line-length = 160 -max-complexity = 10 +[tool.ruff] +line-length = 100 exclude = [ "morango/*/migrations/*", "docs", "morango/sync/operations.py", "morango/sync/backends/postgres.py", "morango/sync/backends/sqlite.py", - "tests/testapp/tests/test_crypto.py" + "tests/testapp/tests/test_crypto.py", +] + +[tool.ruff.lint] +select = [ + "E", # pycodestyle errors + "W", # pycodestyle warnings + "F", # pyflakes + "I", # isort + "T10", # debugger statements (replaces debug-statements hook) + "T20", # print statements (replaces flake8-print plugin) + "C90", # mccabe complexity +] +ignore = [ + "E501", # line too long - formatter handles wrapping where possible ] -ignore = "E203,W503" -[tool.isort] -atomic = true -multi_line_output = 5 -line_length = 160 -indent = " " -combine_as_imports = true -skip = ["wsgi.py", "docs", "env", "cli.py", "test", ".eggs", "build"] +[tool.ruff.lint.isort] +combine-as-imports = true + +[tool.ruff.lint.mccabe] +max-complexity = 10 [tool.coverage.run] branch = true @@ -145,3 +155,6 @@ norecursedirs = ["dist", "tmp*", ".svn", ".*"] markers = [ "windows: mark a test as a windows test" ] + +[tool.uv] +exclude-newer = "7 days" diff --git a/tests/testapp/facility_profile/admin.py b/tests/testapp/facility_profile/admin.py index 8c38f3f3..846f6b40 100644 --- a/tests/testapp/facility_profile/admin.py +++ b/tests/testapp/facility_profile/admin.py @@ -1,3 +1 @@ -from django.contrib import admin - # Register your models here. diff --git a/tests/testapp/facility_profile/apps.py b/tests/testapp/facility_profile/apps.py index dc5c4f31..ac3b9a93 100644 --- a/tests/testapp/facility_profile/apps.py +++ b/tests/testapp/facility_profile/apps.py @@ -2,4 +2,4 @@ class FacilityProfileConfig(AppConfig): - name = 'facility_profile' + name = "facility_profile" diff --git a/tests/testapp/facility_profile/migrations/0001_initial.py b/tests/testapp/facility_profile/migrations/0001_initial.py index cdf6e7f0..642b228a 100644 --- a/tests/testapp/facility_profile/migrations/0001_initial.py +++ b/tests/testapp/facility_profile/migrations/0001_initial.py @@ -4,120 +4,161 @@ import django.db.models.deletion import django.utils.timezone -import facility_profile.models import mptt.fields from django.conf import settings -from django.db import migrations -from django.db import models +from django.db import migrations, models +import facility_profile.models import morango.models.fields.uuids class Migration(migrations.Migration): - initial = True - dependencies = [ - ] + dependencies = [] operations = [ migrations.CreateModel( - name='MyUser', + name="MyUser", fields=[ - ('password', models.CharField(max_length=128, verbose_name='password')), - ('last_login', models.DateTimeField(blank=True, null=True, verbose_name='last login')), - ('id', morango.models.fields.uuids.UUIDField(editable=False, primary_key=True, serialize=False)), - ('_morango_dirty_bit', models.BooleanField(default=True, editable=False)), - ('_morango_source_id', models.CharField(editable=False, max_length=96)), - ('_morango_partition', models.CharField(editable=False, max_length=128)), - ('is_staff', models.BooleanField(default=False)), - ('is_superuser', models.BooleanField(default=False)), - ('username', models.CharField(max_length=20, unique=True)), + ("password", models.CharField(max_length=128, verbose_name="password")), + ( + "last_login", + models.DateTimeField(blank=True, null=True, verbose_name="last login"), + ), + ( + "id", + morango.models.fields.uuids.UUIDField( + editable=False, primary_key=True, serialize=False + ), + ), + ("_morango_dirty_bit", models.BooleanField(default=True, editable=False)), + ("_morango_source_id", models.CharField(editable=False, max_length=96)), + ("_morango_partition", models.CharField(editable=False, max_length=128)), + ("is_staff", models.BooleanField(default=False)), + ("is_superuser", models.BooleanField(default=False)), + ("username", models.CharField(max_length=20, unique=True)), ], options={ - 'abstract': False, + "abstract": False, }, managers=[ - ('objects', facility_profile.models.SyncableUserModelManager()), + ("objects", facility_profile.models.SyncableUserModelManager()), ], ), migrations.CreateModel( - name='Facility', + name="Facility", fields=[ - ('id', morango.models.fields.uuids.UUIDField(editable=False, primary_key=True, serialize=False)), - ('_morango_dirty_bit', models.BooleanField(default=True, editable=False)), - ('_morango_source_id', models.CharField(editable=False, max_length=96)), - ('_morango_partition', models.CharField(editable=False, max_length=128)), - ('name', models.CharField(max_length=100)), - ('now_date', models.DateTimeField(default=django.utils.timezone.now)), - ('lft', models.PositiveIntegerField(db_index=True, editable=False)), - ('rght', models.PositiveIntegerField(db_index=True, editable=False)), - ('tree_id', models.PositiveIntegerField(db_index=True, editable=False)), - ('level', models.PositiveIntegerField(db_index=True, editable=False)), ( - 'parent', mptt.fields.TreeForeignKey( + "id", + morango.models.fields.uuids.UUIDField( + editable=False, primary_key=True, serialize=False + ), + ), + ("_morango_dirty_bit", models.BooleanField(default=True, editable=False)), + ("_morango_source_id", models.CharField(editable=False, max_length=96)), + ("_morango_partition", models.CharField(editable=False, max_length=128)), + ("name", models.CharField(max_length=100)), + ("now_date", models.DateTimeField(default=django.utils.timezone.now)), + ("lft", models.PositiveIntegerField(db_index=True, editable=False)), + ("rght", models.PositiveIntegerField(db_index=True, editable=False)), + ("tree_id", models.PositiveIntegerField(db_index=True, editable=False)), + ("level", models.PositiveIntegerField(db_index=True, editable=False)), + ( + "parent", + mptt.fields.TreeForeignKey( blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, - related_name='children', - to='facility_profile.Facility' - ) + related_name="children", + to="facility_profile.Facility", + ), ), ], options={ - 'abstract': False, + "abstract": False, }, ), migrations.CreateModel( - name='InteractionLog', + name="InteractionLog", fields=[ - ('id', morango.models.fields.uuids.UUIDField(editable=False, primary_key=True, serialize=False)), - ('_morango_dirty_bit', models.BooleanField(default=True, editable=False)), - ('_morango_source_id', models.CharField(editable=False, max_length=96)), - ('_morango_partition', models.CharField(editable=False, max_length=128)), - ('content_id', morango.models.fields.uuids.UUIDField(db_index=True, default=uuid.uuid4)), - ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ( + "id", + morango.models.fields.uuids.UUIDField( + editable=False, primary_key=True, serialize=False + ), + ), + ("_morango_dirty_bit", models.BooleanField(default=True, editable=False)), + ("_morango_source_id", models.CharField(editable=False, max_length=96)), + ("_morango_partition", models.CharField(editable=False, max_length=128)), + ( + "content_id", + morango.models.fields.uuids.UUIDField(db_index=True, default=uuid.uuid4), + ), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL + ), + ), ], options={ - 'abstract': False, + "abstract": False, }, ), migrations.CreateModel( - name='ProxyParent', + name="ProxyParent", fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('kind', models.CharField(max_length=20)), - ('lft', models.PositiveIntegerField(db_index=True, editable=False)), - ('rght', models.PositiveIntegerField(db_index=True, editable=False)), - ('tree_id', models.PositiveIntegerField(db_index=True, editable=False)), - ('level', models.PositiveIntegerField(db_index=True, editable=False)), + ( + "id", + models.AutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + ("kind", models.CharField(max_length=20)), + ("lft", models.PositiveIntegerField(db_index=True, editable=False)), + ("rght", models.PositiveIntegerField(db_index=True, editable=False)), + ("tree_id", models.PositiveIntegerField(db_index=True, editable=False)), + ("level", models.PositiveIntegerField(db_index=True, editable=False)), ], options={ - 'abstract': False, + "abstract": False, }, ), migrations.CreateModel( - name='SummaryLog', + name="SummaryLog", fields=[ - ('id', morango.models.fields.uuids.UUIDField(editable=False, primary_key=True, serialize=False)), - ('_morango_dirty_bit', models.BooleanField(default=True, editable=False)), - ('_morango_source_id', models.CharField(editable=False, max_length=96)), - ('_morango_partition', models.CharField(editable=False, max_length=128)), - ('content_id', morango.models.fields.uuids.UUIDField(db_index=True, default=uuid.uuid4)), - ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ( + "id", + morango.models.fields.uuids.UUIDField( + editable=False, primary_key=True, serialize=False + ), + ), + ("_morango_dirty_bit", models.BooleanField(default=True, editable=False)), + ("_morango_source_id", models.CharField(editable=False, max_length=96)), + ("_morango_partition", models.CharField(editable=False, max_length=128)), + ( + "content_id", + morango.models.fields.uuids.UUIDField(db_index=True, default=uuid.uuid4), + ), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL + ), + ), ], options={ - 'abstract': False, + "abstract": False, }, ), migrations.CreateModel( - name='ProxyModel', - fields=[ - ], + name="ProxyModel", + fields=[], options={ - 'proxy': True, - 'indexes': [], + "proxy": True, + "indexes": [], }, - bases=('facility_profile.proxyparent',), + bases=("facility_profile.proxyparent",), ), ] diff --git a/tests/testapp/facility_profile/migrations/0002_auto_20220427_0449.py b/tests/testapp/facility_profile/migrations/0002_auto_20220427_0449.py index 7b7a231f..4d29f113 100644 --- a/tests/testapp/facility_profile/migrations/0002_auto_20220427_0449.py +++ b/tests/testapp/facility_profile/migrations/0002_auto_20220427_0449.py @@ -2,20 +2,23 @@ # Generated by Django 1.11.29 on 2022-04-27 04:49 import django.db.models.deletion from django.conf import settings -from django.db import migrations -from django.db import models +from django.db import migrations, models class Migration(migrations.Migration): - dependencies = [ - ('facility_profile', '0001_initial'), + ("facility_profile", "0001_initial"), ] operations = [ migrations.AlterField( - model_name='interactionlog', - name='user', - field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL), + model_name="interactionlog", + name="user", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), ), ] diff --git a/tests/testapp/facility_profile/migrations/0003_auto_20240129_2025.py b/tests/testapp/facility_profile/migrations/0003_auto_20240129_2025.py index 6b189635..24b0c268 100644 --- a/tests/testapp/facility_profile/migrations/0003_auto_20240129_2025.py +++ b/tests/testapp/facility_profile/migrations/0003_auto_20240129_2025.py @@ -1,47 +1,45 @@ # Generated by Django 3.2.23 on 2024-01-29 20:25 import django.db.models.deletion -from django.db import migrations -from django.db import models +from django.db import migrations, models class Migration(migrations.Migration): - dependencies = [ - ('facility_profile', '0002_auto_20220427_0449'), + ("facility_profile", "0002_auto_20220427_0449"), ] operations = [ migrations.DeleteModel( - name='ProxyParent', + name="ProxyParent", ), migrations.DeleteModel( - name='ProxyModel', + name="ProxyModel", ), migrations.RemoveField( - model_name='facility', - name='level', + model_name="facility", + name="level", ), migrations.RemoveField( - model_name='facility', - name='lft', + model_name="facility", + name="lft", ), migrations.RemoveField( - model_name='facility', - name='rght', + model_name="facility", + name="rght", ), migrations.RemoveField( - model_name='facility', - name='tree_id', + model_name="facility", + name="tree_id", ), migrations.AlterField( - model_name='facility', - name='parent', + model_name="facility", + name="parent", field=models.ForeignKey( blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, - related_name='children', - to='facility_profile.facility' + related_name="children", + to="facility_profile.facility", ), ), ] diff --git a/tests/testapp/facility_profile/migrations/0004_testmodel.py b/tests/testapp/facility_profile/migrations/0004_testmodel.py index 216ef9c7..36e8bca6 100644 --- a/tests/testapp/facility_profile/migrations/0004_testmodel.py +++ b/tests/testapp/facility_profile/migrations/0004_testmodel.py @@ -1,29 +1,32 @@ # Generated by Django 3.2.25 on 2025-09-22 23:23 -from django.db import migrations -from django.db import models +from django.db import migrations, models import morango.models.fields.uuids class Migration(migrations.Migration): - dependencies = [ - ('facility_profile', '0003_auto_20240129_2025'), + ("facility_profile", "0003_auto_20240129_2025"), ] operations = [ migrations.CreateModel( - name='TestModel', + name="TestModel", fields=[ - ('id', morango.models.fields.uuids.UUIDField(editable=False, primary_key=True, serialize=False)), - ('_morango_dirty_bit', models.BooleanField(default=True, editable=False)), - ('_morango_source_id', models.CharField(editable=False, max_length=96)), - ('_morango_partition', models.CharField(editable=False, max_length=128)), - ('name', models.CharField(max_length=100)), - ('hidden', models.BooleanField(default=False)), + ( + "id", + morango.models.fields.uuids.UUIDField( + editable=False, primary_key=True, serialize=False + ), + ), + ("_morango_dirty_bit", models.BooleanField(default=True, editable=False)), + ("_morango_source_id", models.CharField(editable=False, max_length=96)), + ("_morango_partition", models.CharField(editable=False, max_length=128)), + ("name", models.CharField(max_length=100)), + ("hidden", models.BooleanField(default=False)), ], options={ - 'abstract': False, + "abstract": False, }, ), ] diff --git a/tests/testapp/facility_profile/migrations/0005_conditionallog.py b/tests/testapp/facility_profile/migrations/0005_conditionallog.py index a82b6b4e..2b326658 100644 --- a/tests/testapp/facility_profile/migrations/0005_conditionallog.py +++ b/tests/testapp/facility_profile/migrations/0005_conditionallog.py @@ -3,32 +3,51 @@ import django.db.models.deletion from django.conf import settings -from django.db import migrations -from django.db import models +from django.db import migrations, models import morango.models.fields.uuids class Migration(migrations.Migration): - dependencies = [ - ('facility_profile', '0004_testmodel'), + ("facility_profile", "0004_testmodel"), ] operations = [ migrations.CreateModel( - name='ConditionalLog', + name="ConditionalLog", fields=[ - ('id', morango.models.fields.uuids.UUIDField(editable=False, primary_key=True, serialize=False)), - ('_morango_dirty_bit', models.BooleanField(default=True, editable=False)), - ('_morango_source_id', models.CharField(editable=False, max_length=96)), - ('_morango_partition', models.CharField(editable=False, max_length=128)), - ('content_id', morango.models.fields.uuids.UUIDField(db_index=True, default=uuid.uuid4)), - ('facility', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='facility_profile.facility')), - ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ( + "id", + morango.models.fields.uuids.UUIDField( + editable=False, primary_key=True, serialize=False + ), + ), + ("_morango_dirty_bit", models.BooleanField(default=True, editable=False)), + ("_morango_source_id", models.CharField(editable=False, max_length=96)), + ("_morango_partition", models.CharField(editable=False, max_length=128)), + ( + "content_id", + morango.models.fields.uuids.UUIDField(db_index=True, default=uuid.uuid4), + ), + ( + "facility", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="facility_profile.facility" + ), + ), + ( + "user", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), ], options={ - 'abstract': False, + "abstract": False, }, ), ] diff --git a/tests/testapp/facility_profile/models.py b/tests/testapp/facility_profile/models.py index 7c717cb5..da1d6cab 100644 --- a/tests/testapp/facility_profile/models.py +++ b/tests/testapp/facility_profile/models.py @@ -1,7 +1,6 @@ import uuid -from django.contrib.auth.models import AbstractBaseUser -from django.contrib.auth.models import UserManager +from django.contrib.auth.models import AbstractBaseUser, UserManager from django.db import models from django.utils import timezone @@ -11,7 +10,7 @@ class FacilityDataSyncableModel(SyncableModel): - morango_profile = 'facilitydata' + morango_profile = "facilitydata" class Meta: abstract = True @@ -26,7 +25,14 @@ class Facility(FacilityDataSyncableModel): name = models.CharField(max_length=100) now_date = models.DateTimeField(default=timezone.now) - parent = models.ForeignKey('self', null=True, blank=True, related_name='children', db_index=True, on_delete=models.CASCADE) + parent = models.ForeignKey( + "self", + null=True, + blank=True, + related_name="children", + db_index=True, + on_delete=models.CASCADE, + ) def calculate_source_id(self, *args, **kwargs): return self.name @@ -35,7 +41,7 @@ def calculate_partition(self, *args, **kwargs): if self.id: return uuid.UUID(self.id).hex else: - return '{id}'.format(id=self.ID_PLACEHOLDER) + return "{id}".format(id=self.ID_PLACEHOLDER) def clean_fields(self, *args, **kwargs): # reference parent here just to trigger a non-validation error to make sure we handle it @@ -62,7 +68,7 @@ def calculate_source_id(self, *args, **kwargs): return uuid.uuid5(uuid.UUID("a" * 32), self.username).hex def calculate_partition(self, *args, **kwargs): - return '{id}:user'.format(id=self.ID_PLACEHOLDER) + return "{id}:user".format(id=self.ID_PLACEHOLDER) def has_morango_certificate_scope_permission(self, scope_definition_id, scope_params): return self.is_superuser @@ -79,10 +85,10 @@ class SummaryLog(FacilityDataSyncableModel): content_id = UUIDField(db_index=True, default=uuid.uuid4) def calculate_source_id(self, *args, **kwargs): - return '{}:{}'.format(self.user_id, self.content_id) + return "{}:{}".format(self.user_id, self.content_id) def calculate_partition(self, *args, **kwargs): - return '{user_id}:user:summary'.format(user_id=self.user_id) + return "{user_id}:user:summary".format(user_id=self.user_id) class InteractionLog(FacilityDataSyncableModel): @@ -95,7 +101,7 @@ def calculate_source_id(self, *args, **kwargs): return None def calculate_partition(self, *args, **kwargs): - return '{user_id}:user:interaction'.format(user_id=self.user_id) + return "{user_id}:user:interaction".format(user_id=self.user_id) class ConditionalLog(FacilityDataSyncableModel): @@ -146,4 +152,4 @@ def calculate_source_id(self, *args, **kwargs): return self.name def calculate_partition(self, *args, **kwargs): - return 'testmodel' + return "testmodel" diff --git a/tests/testapp/facility_profile/tests.py b/tests/testapp/facility_profile/tests.py index 7ce503c2..a39b155a 100644 --- a/tests/testapp/facility_profile/tests.py +++ b/tests/testapp/facility_profile/tests.py @@ -1,3 +1 @@ -from django.test import TestCase - # Create your tests here. diff --git a/tests/testapp/facility_profile/views.py b/tests/testapp/facility_profile/views.py index 91ea44a2..60f00ef0 100644 --- a/tests/testapp/facility_profile/views.py +++ b/tests/testapp/facility_profile/views.py @@ -1,3 +1 @@ -from django.shortcuts import render - # Create your views here. diff --git a/tests/testapp/manage.py b/tests/testapp/manage.py index 02bbf70a..0087dbe5 100755 --- a/tests/testapp/manage.py +++ b/tests/testapp/manage.py @@ -2,7 +2,8 @@ import os import sys -import morango # noqa F401 +import morango # noqa F401 + # Import morango to ensure that we do the monkey patching needed # for Django 1.11 to work with Python 3.10+ diff --git a/tests/testapp/testapp/cgi.py b/tests/testapp/testapp/cgi.py index a825ef5a..95a2abde 100644 --- a/tests/testapp/testapp/cgi.py +++ b/tests/testapp/testapp/cgi.py @@ -4,6 +4,7 @@ Informed by the PR that removed its use in Django: https://github.com/django/django/pull/15679 """ + from django.utils.regex_helper import _lazy_re_compile diff --git a/tests/testapp/testapp/db.py b/tests/testapp/testapp/db.py index 40d1abe4..46d51cb0 100644 --- a/tests/testapp/testapp/db.py +++ b/tests/testapp/testapp/db.py @@ -6,6 +6,7 @@ class TestingRouter(object): A router to control all database operations on models in the test application. """ + def db_for_read(self, *args, **kwargs): return os.environ.get("MORANGO_TEST_DATABASE", "default") diff --git a/tests/testapp/testapp/postgres_settings.py b/tests/testapp/testapp/postgres_settings.py index 535b3a30..5a3a8d5f 100644 --- a/tests/testapp/testapp/postgres_settings.py +++ b/tests/testapp/testapp/postgres_settings.py @@ -1,18 +1,17 @@ """ A settings module for running tests using a postgres db backend. """ + from .settings import * # noqa DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.postgresql', - 'HOST': 'localhost', - 'USER': 'postgres', - 'PASSWORD': 'postgres', - 'NAME': 'default', # This module should never be used outside of tests -- so this name is irrelevant - 'TEST': { - 'NAME': 'test' - } + "default": { + "ENGINE": "django.db.backends.postgresql", + "HOST": "localhost", + "USER": "postgres", + "PASSWORD": "postgres", + "NAME": "default", # This module should never be used outside of tests -- so this name is irrelevant + "TEST": {"NAME": "test"}, }, } diff --git a/tests/testapp/testapp/server2_settings.py b/tests/testapp/testapp/server2_settings.py index 8aed41a0..c7473347 100644 --- a/tests/testapp/testapp/server2_settings.py +++ b/tests/testapp/testapp/server2_settings.py @@ -1,5 +1,5 @@ -from .settings import * # noqa +from .settings import * # noqa DATABASES = { - 'default': DATABASES["default2"], # noqa + "default": DATABASES["default2"], # noqa } diff --git a/tests/testapp/testapp/settings.py b/tests/testapp/testapp/settings.py index f9a19db4..df4d95d6 100644 --- a/tests/testapp/testapp/settings.py +++ b/tests/testapp/testapp/settings.py @@ -9,6 +9,7 @@ For the full list of settings and their values, see https://docs.djangoproject.com/en/1.9/ref/settings/ """ + import os # Build paths inside the project like this: os.path.join(BASE_DIR, ...) @@ -19,7 +20,7 @@ # See https://docs.djangoproject.com/en/1.9/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = '-#()od3p8n@o&9kcj(s63!#^tziq+j!nuwlyptw#o06t&wrk$q' +SECRET_KEY = "-#()od3p8n@o&9kcj(s63!#^tziq+j!nuwlyptw#o06t&wrk$q" # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True @@ -30,65 +31,65 @@ # Application definition INSTALLED_APPS = [ - 'django.contrib.admin', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', - 'rest_framework', - 'morango', - 'facility_profile', + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", + "rest_framework", + "morango", + "facility_profile", ] MIDDLEWARE = [ - 'django.middleware.security.SecurityMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", ] -ROOT_URLCONF = 'testapp.urls' +ROOT_URLCONF = "testapp.urls" TEMPLATES = [ { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", ], }, }, ] -WSGI_APPLICATION = 'testapp.wsgi.application' +WSGI_APPLICATION = "testapp.wsgi.application" # Database # https://docs.djangoproject.com/en/1.9/ref/settings/#databases DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(BASE_DIR, "testapp.db"), + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": os.path.join(BASE_DIR, "testapp.db"), }, - 'default2': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(BASE_DIR, "testapp2.db"), - 'TEST': { + "default2": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": os.path.join(BASE_DIR, "testapp2.db"), + "TEST": { # Force using a disk based database for testing # so that we can share it between processes. - 'NAME': os.path.join(BASE_DIR, "testapp2.db"), - } - } + "NAME": os.path.join(BASE_DIR, "testapp2.db"), + }, + }, } @@ -104,16 +105,16 @@ AUTH_PASSWORD_VALIDATORS = [ { - 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", }, { - 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", }, { - 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", }, { - 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", }, ] @@ -121,9 +122,9 @@ # Internationalization # https://docs.djangoproject.com/en/1.9/topics/i18n/ -LANGUAGE_CODE = 'en-us' +LANGUAGE_CODE = "en-us" -TIME_ZONE = 'UTC' +TIME_ZONE = "UTC" USE_I18N = True @@ -135,6 +136,6 @@ # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/1.9/howto/static-files/ -STATIC_URL = '/static/' +STATIC_URL = "/static/" -AUTH_USER_MODEL = 'facility_profile.MyUser' +AUTH_USER_MODEL = "facility_profile.MyUser" diff --git a/tests/testapp/testapp/urls.py b/tests/testapp/testapp/urls.py index 54f02ad2..97712e08 100644 --- a/tests/testapp/testapp/urls.py +++ b/tests/testapp/testapp/urls.py @@ -13,9 +13,9 @@ 1. Import the include() function: from django.conf.urls import url, include 2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) """ -from django.urls import include -from django.urls import path + +from django.urls import include, path urlpatterns = [ - path('', include('morango.urls')), + path("", include("morango.urls")), ] diff --git a/tests/testapp/tests/helpers.py b/tests/testapp/tests/helpers.py index 1dbe1168..d5a3f90e 100644 --- a/tests/testapp/tests/helpers.py +++ b/tests/testapp/tests/helpers.py @@ -1,6 +1,7 @@ """ Helper functions for use across syncing related functionality. """ + import json import uuid @@ -12,28 +13,25 @@ from django.db.migrations.recorder import MigrationRecorder from django.test.testcases import LiveServerTestCase from django.utils import timezone -from facility_profile.models import Facility -from facility_profile.models import InteractionLog -from facility_profile.models import MyUser -from facility_profile.models import SummaryLog +from facility_profile.models import Facility, InteractionLog, MyUser, SummaryLog -from .compat import EnvironmentVarGuard from morango.api.serializers import BufferSerializer -from morango.models.core import AbstractStore -from morango.models.core import Buffer -from morango.models.core import DatabaseIDModel -from morango.models.core import InstanceIDModel -from morango.models.core import RecordMaxCounter -from morango.models.core import RecordMaxCounterBuffer -from morango.models.core import Store -from morango.models.core import SyncSession -from morango.models.core import TransferSession +from morango.models.core import ( + AbstractStore, + Buffer, + DatabaseIDModel, + InstanceIDModel, + RecordMaxCounter, + RecordMaxCounterBuffer, + Store, + SyncSession, + TransferSession, +) from morango.sync.context import SessionContext -from morango.sync.controller import MorangoProfileController -from morango.sync.controller import SessionController -from morango.sync.syncsession import NetworkSyncConnection -from morango.sync.syncsession import SyncSessionClient -from morango.sync.syncsession import TransferClient +from morango.sync.controller import MorangoProfileController, SessionController +from morango.sync.syncsession import NetworkSyncConnection, SyncSessionClient, TransferClient + +from .compat import EnvironmentVarGuard class FacilityModelFactory(factory.django.DjangoModelFactory): @@ -79,12 +77,10 @@ def serialized_facility_factory(identifier): def create_dummy_store_data(): data = {} DatabaseIDModel.objects.create() - data["group1_id"] = InstanceIDModel.get_or_create_current_instance()[ - 0 - ] # counter is at 0 + data["group1_id"] = InstanceIDModel.get_or_create_current_instance()[0] # counter is at 0 # create controllers for app/store/buffer operations - conn = mock.Mock(spec='morango.sync.syncsession.NetworkSyncConnection') + conn = mock.Mock(spec="morango.sync.syncsession.NetworkSyncConnection") conn.server_info = dict(capabilities=[]) data["mc"] = MorangoProfileController("facilitydata") data["sc"] = TransferClient(conn, "host", SessionController.build()) @@ -110,9 +106,7 @@ def create_dummy_store_data(): # create users and logs associated with user data["user1"] = MyUser.objects.create(username="bob") - data["user1_sumlogs"] = [ - SummaryLog.objects.create(user=data["user1"]) for _ in range(5) - ] + data["user1_sumlogs"] = [SummaryLog.objects.create(user=data["user1"]) for _ in range(5)] data["mc"].serialize_into_store() # counter is at 3 @@ -120,9 +114,7 @@ def create_dummy_store_data(): with EnvironmentVarGuard() as env: env["MORANGO_SYSTEM_ID"] = "new_sys_id" - data["group2_id"] = InstanceIDModel.get_or_create_current_instance( - clear_cache=True - )[ + data["group2_id"] = InstanceIDModel.get_or_create_current_instance(clear_cache=True)[ 0 ] # new counter is at 0 @@ -131,26 +123,20 @@ def create_dummy_store_data(): # create users and logs associated with user data["user2"] = MyUser.objects.create(username="rob") - data["user2_sumlogs"] = [ - SummaryLog.objects.create(user=data["user2"]) for _ in range(5) - ] + data["user2_sumlogs"] = [SummaryLog.objects.create(user=data["user2"]) for _ in range(5)] data["user2_interlogs"] = [ InteractionLog.objects.create(user=data["user2"]) for _ in range(5) ] data["user3"] = MyUser.objects.create(username="zob") - data["user3_sumlogs"] = [ - SummaryLog.objects.create(user=data["user3"]) for _ in range(5) - ] + data["user3_sumlogs"] = [SummaryLog.objects.create(user=data["user3"]) for _ in range(5)] data["user3_interlogs"] = [ InteractionLog.objects.create(user=data["user3"]) for _ in range(5) ] data["mc"].serialize_into_store() # new counter is at 2 - data["user4"] = MyUser.objects.create( - username="invalid", _morango_partition="badpartition" - ) + data["user4"] = MyUser.objects.create(username="invalid", _morango_partition="badpartition") data["mc"].serialize_into_store() # new counter is at 3 return data @@ -199,9 +185,7 @@ def create_buffer_and_store_dummy_data(transfer_session_id): model_uuid=data["model1"], transfer_session_id=transfer_session_id, ) - create_rmcb_data( - 1, 2, 3, 4, data["model1_rmcb_ids"], data["model1"], transfer_session_id - ) + create_rmcb_data(1, 2, 3, 4, data["model1_rmcb_ids"], data["model1"], transfer_session_id) # example data for merge conflict (rmcb.counter > rmc.counter) data["model2"] = uuid.uuid4().hex @@ -227,9 +211,7 @@ def create_buffer_and_store_dummy_data(transfer_session_id): transfer_session_id=transfer_session_id, deleted=1, ) - create_rmcb_data( - 3, 2, 3, 4, data["model2_rmcb_ids"], data["model2"], transfer_session_id - ) + create_rmcb_data(3, 2, 3, 4, data["model2_rmcb_ids"], data["model2"], transfer_session_id) # example data for merge conflict (rmcb.counter <= rmc.counter) data["model5"] = uuid.uuid4().hex @@ -254,9 +236,7 @@ def create_buffer_and_store_dummy_data(transfer_session_id): model_uuid=data["model5"], transfer_session_id=transfer_session_id, ) - create_rmcb_data( - 1, 2, 3, 4, data["model5_rmcb_ids"], data["model5"], transfer_session_id - ) + create_rmcb_data(1, 2, 3, 4, data["model5_rmcb_ids"], data["model5"], transfer_session_id) # example data for merge conflict with hard delete(rmcb.counter <= rmc.counter) data["model7"] = uuid.uuid4().hex @@ -282,9 +262,7 @@ def create_buffer_and_store_dummy_data(transfer_session_id): transfer_session_id=transfer_session_id, hard_deleted=True, ) - create_rmcb_data( - 1, 2, 3, 4, data["model7_rmcb_ids"], data["model7"], transfer_session_id - ) + create_rmcb_data(1, 2, 3, 4, data["model7_rmcb_ids"], data["model7"], transfer_session_id) # example data for ff data["model3"] = uuid.uuid4().hex @@ -308,9 +286,7 @@ def create_buffer_and_store_dummy_data(transfer_session_id): model_uuid=data["model3"], transfer_session_id=transfer_session_id, ) - create_rmcb_data( - 3, 2, 3, 4, data["model3_rmcb_ids"], data["model3"], transfer_session_id - ) + create_rmcb_data(3, 2, 3, 4, data["model3_rmcb_ids"], data["model3"], transfer_session_id) # example for missing store data data["model4"] = uuid.uuid4().hex @@ -322,9 +298,7 @@ def create_buffer_and_store_dummy_data(transfer_session_id): model_uuid=data["model4"], transfer_session_id=transfer_session_id, ) - create_rmcb_data( - 1, 2, 3, 4, data["model4_rmcb_ids"], data["model4"], transfer_session_id - ) + create_rmcb_data(1, 2, 3, 4, data["model4_rmcb_ids"], data["model4"], transfer_session_id) # buffer record with different transfer session id session = SyncSession.objects.create( @@ -345,9 +319,7 @@ def create_buffer_and_store_dummy_data(transfer_session_id): model_uuid=data["model6"], transfer_session_id=data["tfs_id"], ) - create_rmcb_data( - 1, 2, 3, 4, data["model6_rmcb_ids"], data["model6"], data["tfs_id"] - ) + create_rmcb_data(1, 2, 3, 4, data["model6_rmcb_ids"], data["model6"], data["tfs_id"]) return data @@ -385,9 +357,7 @@ def build_buffer_items(self, transfer_session, **kwargs): "partition": kwargs.get("partition", "partition"), "source_id": kwargs.get("source_id", uuid.uuid4().hex), "model_name": kwargs.get("model_name", "contentsummarylog"), - "conflicting_serialized_data": kwargs.get( - "conflicting_serialized_data", "" - ), + "conflicting_serialized_data": kwargs.get("conflicting_serialized_data", ""), "model_uuid": kwargs.get("model_uuid", None), "transfer_session": transfer_session, } @@ -399,9 +369,7 @@ def build_buffer_items(self, transfer_session, **kwargs): ) Buffer.objects.create(**data) - buffered_items = Buffer.objects.filter( - transfer_session=transfer_session - ) + buffered_items = Buffer.objects.filter(transfer_session=transfer_session) serialized_records = BufferSerializer(buffered_items, many=True) return json.dumps(serialized_records.data) @@ -418,7 +386,9 @@ def build_client(self, client_class=TransferClient, controller=None, update_cont records_total=3, ) - client = super(BaseTransferClientTestCase, self).build_client(client_class=client_class, controller=controller) + client = super(BaseTransferClientTestCase, self).build_client( + client_class=client_class, controller=controller + ) if client.context.is_push is None: client.context.update(is_push=self.transfer_session.push) if update_context: @@ -463,10 +433,10 @@ def setUpClass(cls): cls.latest_migration = (cls.app, latest_migration.name) def setUp(self): - assert ( - self.migrate_from and self.migrate_to - ), "TestCase '{}' must define migrate_from and migrate_to properties".format( - type(self).__name__ + assert self.migrate_from and self.migrate_to, ( + "TestCase '{}' must define migrate_from and migrate_to properties".format( + type(self).__name__ + ) ) migrate_from = [(self.app, self.migrate_from)] diff --git a/tests/testapp/tests/integration/test_serialization.py b/tests/testapp/tests/integration/test_serialization.py index e8af1394..23626297 100644 --- a/tests/testapp/tests/integration/test_serialization.py +++ b/tests/testapp/tests/integration/test_serialization.py @@ -6,7 +6,6 @@ class SerializationTestCase(TestCase): - def setUp(self): self.bob = Facility.objects.create(name="bob") self.student = Facility.objects.create(name="student") @@ -14,25 +13,27 @@ def setUp(self): self.student_dict = self.student.serialize() def test_serialization(self): - self.assertEqual(self.bob_dict['name'], 'bob') + self.assertEqual(self.bob_dict["name"], "bob") self.assertEqual(self.bob.morango_model_name, Facility.morango_model_name) def test_field_deserialization(self): - class_model = syncable_models.get_model('facilitydata', self.bob.morango_model_name) + class_model = syncable_models.get_model("facilitydata", self.bob.morango_model_name) self.bob_copy = class_model.deserialize(self.bob_dict) for f in Facility._meta.concrete_fields: # we remove DateTimeField (for now) from this test because serializing and deserializing loses units of precision - if isinstance(f, models.DateTimeField) or \ - f.attname in class_model._morango_internal_fields_not_to_serialize: + if ( + isinstance(f, models.DateTimeField) + or f.attname in class_model._morango_internal_fields_not_to_serialize + ): continue self.assertEqual(getattr(self.bob, f.attname), getattr(self.bob_copy, f.attname)) def test_serializing_different_models(self): - self.assertNotEqual(self.bob_dict['id'], self.student_dict['id']) - self.assertNotEqual(self.bob_dict['name'], self.student_dict['name']) + self.assertNotEqual(self.bob_dict["id"], self.student_dict["id"]) + self.assertNotEqual(self.bob_dict["name"], self.student_dict["name"]) def test_fields_not_to_serialize(self): - self.assertTrue('now_date' in self.bob_dict) + self.assertTrue("now_date" in self.bob_dict) Facility._fields_not_to_serialize = ("now_date",) self.bob_dict = self.bob.serialize() - self.assertFalse('now_data' in self.bob_dict) + self.assertFalse("now_data" in self.bob_dict) diff --git a/tests/testapp/tests/integration/test_signals.py b/tests/testapp/tests/integration/test_signals.py index e245a271..461a3113 100644 --- a/tests/testapp/tests/integration/test_signals.py +++ b/tests/testapp/tests/integration/test_signals.py @@ -1,18 +1,17 @@ from django.test import TestCase from facility_profile.models import Facility -from ..helpers import FacilityModelFactory -from morango.models.core import DeletedModels -from morango.models.core import InstanceIDModel +from morango.models.core import DeletedModels, InstanceIDModel from morango.sync.controller import MorangoProfileController +from ..helpers import FacilityModelFactory + class PostDeleteSignalsTestCase(TestCase): - def setUp(self): InstanceIDModel.get_or_create_current_instance() [FacilityModelFactory() for _ in range(10)] - self.mc = MorangoProfileController('facilitydata') + self.mc = MorangoProfileController("facilitydata") self.mc.serialize_into_store() def test_deleted_flag_gets_set(self): diff --git a/tests/testapp/tests/integration/test_syncing_models.py b/tests/testapp/tests/integration/test_syncing_models.py index 460b1029..7086fb67 100644 --- a/tests/testapp/tests/integration/test_syncing_models.py +++ b/tests/testapp/tests/integration/test_syncing_models.py @@ -1,8 +1,7 @@ import json from django.test import TestCase -from facility_profile.models import MyUser -from facility_profile.models import TestModel +from facility_profile.models import MyUser, TestModel from morango.models.core import Store from morango.models.manager import SyncableModelManager @@ -11,9 +10,8 @@ class SyncingModelsTestCase(TestCase): - def setUp(self): - MyUser.objects.create(username='beans') + MyUser.objects.create(username="beans") def test_syncable_manager_inheritance(self): self.assertTrue(isinstance(MyUser.objects, SyncableModelManager)) @@ -81,7 +79,7 @@ def test_syncing_objects_manager_with_custom_default_manager(self): # Verify that syncing_objects manager includes all objects for syncing self.assertEqual(TestModel.syncing_objects.count(), 2) - syncing_names = set(TestModel.syncing_objects.values_list('name', flat=True)) + syncing_names = set(TestModel.syncing_objects.values_list("name", flat=True)) self.assertEqual(syncing_names, {"visible", "hidden"}) def test_hidden_models_serialization_into_store(self): @@ -105,12 +103,12 @@ def test_hidden_models_serialization_into_store(self): serialized_names = set() for store_record in store_records: serialized_data = json.loads(store_record.serialized) - serialized_names.add(serialized_data['name']) + serialized_names.add(serialized_data["name"]) self.assertEqual(serialized_names, {"visible", "hidden"}) # Verify the store records have the correct IDs - store_ids = set(store_records.values_list('id', flat=True)) + store_ids = set(store_records.values_list("id", flat=True)) expected_ids = {str(visible_obj.id), str(hidden_obj.id)} self.assertEqual(store_ids, expected_ids) diff --git a/tests/testapp/tests/integration/test_syncsession.py b/tests/testapp/tests/integration/test_syncsession.py index 5e7be2d2..c815e4c9 100644 --- a/tests/testapp/tests/integration/test_syncsession.py +++ b/tests/testapp/tests/integration/test_syncsession.py @@ -11,24 +11,16 @@ import requests from django.conf import settings from django.test.testcases import TransactionTestCase -from facility_profile.models import InteractionLog -from facility_profile.models import MyUser -from facility_profile.models import SummaryLog -from requests.exceptions import RequestException -from requests.exceptions import Timeout +from facility_profile.models import InteractionLog, MyUser, SummaryLog +from requests.exceptions import RequestException, Timeout from testapp.settings import BASE_DIR -from ..compat import EnvironmentVarGuard from morango.errors import MorangoError -from morango.models.certificates import Certificate -from morango.models.certificates import Filter -from morango.models.certificates import Key -from morango.models.certificates import ScopeDefinition -from morango.models.core import Buffer -from morango.models.core import InstanceIDModel -from morango.models.core import TransferSession +from morango.models.certificates import Certificate, Filter, Key, ScopeDefinition +from morango.models.core import Buffer, InstanceIDModel, TransferSession from morango.sync.controller import MorangoProfileController +from ..compat import EnvironmentVarGuard SECOND_TEST_DATABASE = "default2" SECOND_SYSTEM_ID = "default2" @@ -68,7 +60,16 @@ def baseurl(self): def start(self): manage_py_path = os.path.join(BASE_DIR, "manage.py") self._instance = subprocess.Popen( - [sys.executable, manage_py_path, "runserver", "--nothreading", "--noreload", "--settings", "testapp.server2_settings", f"{self.host}:{self.port}"], + [ + sys.executable, + manage_py_path, + "runserver", + "--nothreading", + "--noreload", + "--settings", + "testapp.server2_settings", + f"{self.host}:{self.port}", + ], env=self.env, ) self._wait_for_server_start() @@ -92,9 +93,7 @@ def kill(self): pass -@pytest.mark.skipif( - getattr(settings, "MORANGO_TEST_POSTGRESQL", False), reason="Not supported" -) +@pytest.mark.skipif(getattr(settings, "MORANGO_TEST_POSTGRESQL", False), reason="Not supported") class PushPullClientTestCase(TransactionTestCase): profile = "facilitydata" databases = ["default", SECOND_TEST_DATABASE] @@ -108,7 +107,7 @@ def setUpClass(cls): def tearDownClass(cls): # There may not be a 'server' attribute if setUpClass() for some # reasons has raised an exception. - if hasattr(cls, 'server'): + if hasattr(cls, "server"): # Terminate the live server's thread cls.server.kill() super(TransactionTestCase, cls).tearDownClass() @@ -116,9 +115,7 @@ def tearDownClass(cls): def setUp(self): super(PushPullClientTestCase, self).setUp() self.profile_controller = MorangoProfileController(self.profile) - self.conn = self.profile_controller.create_network_connection( - self.server.baseurl - ) + self.conn = self.profile_controller.create_network_connection(self.server.baseurl) self.conn.chunk_size = 3 self.remote_user, self.root_cert_id = self._setUpServer() @@ -210,9 +207,7 @@ def assertLastActivityUpdate(self, transfer_session=None): transfer_session.sync_session.last_activity_timestamp, ) self.last_transfer_activity = transfer_session.last_activity_timestamp - self.last_session_activity = ( - transfer_session.sync_session.last_activity_timestamp - ) + self.last_session_activity = transfer_session.sync_session.last_activity_timestamp def test_push(self): for _ in range(5): @@ -220,12 +215,8 @@ def test_push(self): InteractionLog.objects.create(user=self.local_user) with second_environment(): - self.assertEqual( - 0, SummaryLog.objects.filter(user=self.remote_user).count() - ) - self.assertEqual( - 0, InteractionLog.objects.filter(user=self.remote_user).count() - ) + self.assertEqual(0, SummaryLog.objects.filter(user=self.remote_user).count()) + self.assertEqual(0, InteractionLog.objects.filter(user=self.remote_user).count()) client = self.client.get_push_client() client.signals.queuing.completed.connect(self.assertLastActivityUpdate) @@ -238,24 +229,16 @@ def test_push(self): transfer_session = client.context.transfer_session self.assertNotEqual(0, transfer_session.records_total) self.assertEqual(0, transfer_session.records_transferred) - self.assertLessEqual( - 1, Buffer.objects.filter(transfer_session=transfer_session).count() - ) + self.assertLessEqual(1, Buffer.objects.filter(transfer_session=transfer_session).count()) client.run() self.assertNotEqual(0, transfer_session.records_transferred) client.finalize() - self.assertEqual( - 0, Buffer.objects.filter(transfer_session=transfer_session).count() - ) + self.assertEqual(0, Buffer.objects.filter(transfer_session=transfer_session).count()) self.assertEqual(0, TransferSession.objects.filter(active=True).count()) with second_environment(): - self.assertEqual( - 5, SummaryLog.objects.filter(user=self.remote_user).count() - ) - self.assertEqual( - 5, InteractionLog.objects.filter(user=self.remote_user).count() - ) + self.assertEqual(5, SummaryLog.objects.filter(user=self.remote_user).count()) + self.assertEqual(5, InteractionLog.objects.filter(user=self.remote_user).count()) def test_pull(self): with second_environment(): @@ -279,13 +262,9 @@ def test_pull(self): self.assertEqual(0, transfer_session.records_transferred) client.run() self.assertNotEqual(0, transfer_session.records_transferred) - self.assertLessEqual( - 1, Buffer.objects.filter(transfer_session=transfer_session).count() - ) + self.assertLessEqual(1, Buffer.objects.filter(transfer_session=transfer_session).count()) client.finalize() - self.assertEqual( - 0, Buffer.objects.filter(transfer_session=transfer_session).count() - ) + self.assertEqual(0, Buffer.objects.filter(transfer_session=transfer_session).count()) self.assertEqual(0, TransferSession.objects.filter(active=True).count()) self.assertEqual(5, SummaryLog.objects.filter(user=self.local_user).count()) diff --git a/tests/testapp/tests/models/test_certificates.py b/tests/testapp/tests/models/test_certificates.py index f419599e..551dbd5c 100644 --- a/tests/testapp/tests/models/test_certificates.py +++ b/tests/testapp/tests/models/test_certificates.py @@ -1,21 +1,18 @@ import json -from django.test import SimpleTestCase -from django.test import TestCase +from django.test import SimpleTestCase, TestCase -from morango.errors import CertificateIDInvalid -from morango.errors import CertificateProfileInvalid -from morango.errors import CertificateRootScopeInvalid -from morango.errors import CertificateScopeNotSubset -from morango.errors import CertificateSignatureInvalid -from morango.models.certificates import Certificate -from morango.models.certificates import Filter -from morango.models.certificates import Key -from morango.models.certificates import ScopeDefinition +from morango.errors import ( + CertificateIDInvalid, + CertificateProfileInvalid, + CertificateRootScopeInvalid, + CertificateScopeNotSubset, + CertificateSignatureInvalid, +) +from morango.models.certificates import Certificate, Filter, Key, ScopeDefinition class CertificateTestCaseMixin(object): - def setUp(self): self.profile = "testprofile" @@ -49,7 +46,9 @@ def setUp(self): profile=self.profile, scope_definition=self.subset_scope_def, scope_version=self.subset_scope_def.version, - scope_params=json.dumps({"mainpartition": self.root_cert.id, "subpartition": "abracadabra"}), + scope_params=json.dumps( + {"mainpartition": self.root_cert.id, "subpartition": "abracadabra"} + ), private_key=Key(), ) self.root_cert.sign_certificate(self.subset_cert) @@ -57,7 +56,6 @@ def setUp(self): class CertificateCheckingTestCase(CertificateTestCaseMixin, TestCase): - def test_good_certificates_validate(self): self.root_cert.check_certificate() @@ -114,11 +112,14 @@ def test_bad_root_scope_does_not_validate(self): class CertificateSerializationTestCase(CertificateTestCaseMixin, TestCase): - def setUp(self): super(CertificateSerializationTestCase, self).setUp() - self.root_cert_deserialized = Certificate.deserialize(self.root_cert.serialized, self.root_cert.signature) - self.subset_cert_deserialized = Certificate.deserialize(self.subset_cert.serialized, self.subset_cert.signature) + self.root_cert_deserialized = Certificate.deserialize( + self.root_cert.serialized, self.root_cert.signature + ) + self.subset_cert_deserialized = Certificate.deserialize( + self.subset_cert.serialized, self.subset_cert.signature + ) def test_deserialized_certs_validate(self): self.subset_cert_deserialized.check_certificate() @@ -131,8 +132,12 @@ def test_deserialized_certs_validate(self): self.root_cert_deserialized.full_clean() def test_deserialized_cert_signatures_verify(self): - self.assertTrue(self.root_cert_deserialized.verify("testval", self.root_cert.sign("testval"))) - self.assertTrue(self.subset_cert_deserialized.verify("testval", self.subset_cert.sign("testval"))) + self.assertTrue( + self.root_cert_deserialized.verify("testval", self.root_cert.sign("testval")) + ) + self.assertTrue( + self.subset_cert_deserialized.verify("testval", self.subset_cert.sign("testval")) + ) def test_deserialized_certs_can_be_saved(self): Certificate.objects.all().delete() @@ -141,7 +146,6 @@ def test_deserialized_certs_can_be_saved(self): class CertificateKeySettingTestCase(TestCase): - def test_setting_private_key_sets_public_key(self): cert = Certificate() cert.private_key = Key() diff --git a/tests/testapp/tests/models/test_core.py b/tests/testapp/tests/models/test_core.py index f760eb25..b025ad65 100644 --- a/tests/testapp/tests/models/test_core.py +++ b/tests/testapp/tests/models/test_core.py @@ -2,23 +2,17 @@ import factory import mock -from django.test import override_settings -from django.test import TestCase +from django.test import TestCase, override_settings from django.utils import timezone -from facility_profile.models import Facility -from facility_profile.models import MyUser +from facility_profile.models import Facility, MyUser -from ..helpers import RecordMaxCounterFactory -from ..helpers import StoreFactory -from morango.constants import transfer_stages -from morango.constants import transfer_statuses +from morango.constants import transfer_stages, transfer_statuses from morango.models.certificates import Filter -from morango.models.core import DatabaseMaxCounter -from morango.models.core import Store -from morango.models.core import SyncSession -from morango.models.core import TransferSession +from morango.models.core import DatabaseMaxCounter, Store, SyncSession, TransferSession from morango.sync.controller import MorangoProfileController +from ..helpers import RecordMaxCounterFactory, StoreFactory + class DatabaseMaxCounterFactory(factory.django.DjangoModelFactory): class Meta: @@ -38,9 +32,7 @@ def setUp(self): self.user2_prefix_b = "BBB:user_id:emily" # instance A dmc - DatabaseMaxCounterFactory( - instance_id=self.instance_a, partition=self.prefix_a, counter=15 - ) + DatabaseMaxCounterFactory(instance_id=self.instance_a, partition=self.prefix_a, counter=15) DatabaseMaxCounterFactory( instance_id=self.instance_a, partition=self.user_prefix_a, counter=20 ) @@ -52,9 +44,7 @@ def setUp(self): DatabaseMaxCounterFactory( instance_id=self.instance_b, partition=self.user_prefix_a, counter=10 ) - DatabaseMaxCounterFactory( - instance_id=self.instance_b, partition=self.prefix_b, counter=12 - ) + DatabaseMaxCounterFactory(instance_id=self.instance_b, partition=self.prefix_b, counter=12) DatabaseMaxCounterFactory( instance_id=self.instance_b, partition=self.user_prefix_b, counter=5 ) @@ -66,9 +56,7 @@ def setUp(self): @override_settings(MORANGO_DISABLE_FSIC_V2_FORMAT=True) class OldFilterMaxCounterTestCase(BaseDatabaseMaxCounterTestCase): def test_filter_not_in_dmc(self): - fmcs = DatabaseMaxCounter.calculate_filter_specific_instance_counters( - Filter("ZZZ") - ) + fmcs = DatabaseMaxCounter.calculate_filter_specific_instance_counters(Filter("ZZZ")) self.assertEqual(fmcs, {}) def test_instances_for_one_partition_but_not_other(self): @@ -131,17 +119,13 @@ def test_update_all_fsics(self): def test_update_some_fsics(self): client_fsic = {"a" * 32: 1, "e" * 32: 2, "c" * 32: 1} server_fsic = {"a" * 32: 2, "b" * 32: 1, "c" * 32: 2} - self.assertFalse( - DatabaseMaxCounter.objects.filter(instance_id="e" * 32).exists() - ) + self.assertFalse(DatabaseMaxCounter.objects.filter(instance_id="e" * 32).exists()) for instance_id, counter in server_fsic.items(): DatabaseMaxCounter.objects.create( instance_id=instance_id, counter=counter, partition=self.filter ) DatabaseMaxCounter.update_fsics(client_fsic, Filter(self.filter)) - self.assertTrue( - DatabaseMaxCounter.objects.filter(instance_id="e" * 32).exists() - ) + self.assertTrue(DatabaseMaxCounter.objects.filter(instance_id="e" * 32).exists()) def test_no_fsics_get_updated(self): client_fsic = {"a" * 32: 1, "b" * 32: 1, "c" * 32: 1} @@ -215,7 +199,9 @@ def setUpRecordMaxCounters(self): ) def test_get_instance_counters_for_partitions__producer(self): - counters = DatabaseMaxCounter.get_instance_counters_for_partitions([self.user_prefix_a], is_producer=True) + counters = DatabaseMaxCounter.get_instance_counters_for_partitions( + [self.user_prefix_a], is_producer=True + ) self.assertEqual(1, len(counters)) partition_counters = counters.get(self.user_prefix_a) self.assertEqual(1, len(partition_counters)) @@ -223,7 +209,9 @@ def test_get_instance_counters_for_partitions__producer(self): @override_settings(MORANGO_DISABLE_FSIC_REDUCTION=True) def test_get_instance_counters_for_partitions__producer__no_reduction(self): - counters = DatabaseMaxCounter.get_instance_counters_for_partitions([self.user_prefix_a], is_producer=True) + counters = DatabaseMaxCounter.get_instance_counters_for_partitions( + [self.user_prefix_a], is_producer=True + ) self.assertEqual(1, len(counters)) partition_counters = counters.get(self.user_prefix_a) self.assertEqual(2, len(partition_counters)) @@ -277,9 +265,7 @@ def test_update_state(self): self.assertEqual(transfer_stages.QUEUING, self.instance.transfer_stage) self.assertEqual(transfer_statuses.PENDING, self.instance.transfer_stage_status) self.assertLess(previous_activity, self.instance.last_activity_timestamp) - self.assertLess( - previous_sync_activity, self.sync_session.last_activity_timestamp - ) + self.assertLess(previous_sync_activity, self.sync_session.last_activity_timestamp) def test_update_state__only_stage(self): self.assertIsNone(self.instance.transfer_stage) @@ -292,9 +278,7 @@ def test_update_state__only_stage(self): self.assertEqual(transfer_stages.QUEUING, self.instance.transfer_stage) self.assertIsNone(self.instance.transfer_stage_status) self.assertLess(previous_activity, self.instance.last_activity_timestamp) - self.assertLess( - previous_sync_activity, self.sync_session.last_activity_timestamp - ) + self.assertLess(previous_sync_activity, self.sync_session.last_activity_timestamp) def test_update_state__only_status(self): self.assertIsNone(self.instance.transfer_stage) @@ -307,9 +291,7 @@ def test_update_state__only_status(self): self.assertIsNone(self.instance.transfer_stage) self.assertEqual(transfer_statuses.PENDING, self.instance.transfer_stage_status) self.assertLess(previous_activity, self.instance.last_activity_timestamp) - self.assertLess( - previous_sync_activity, self.sync_session.last_activity_timestamp - ) + self.assertLess(previous_sync_activity, self.sync_session.last_activity_timestamp) def test_update_state__none(self): self.assertIsNone(self.instance.transfer_stage) @@ -322,9 +304,7 @@ def test_update_state__none(self): self.assertIsNone(self.instance.transfer_stage) self.assertIsNone(self.instance.transfer_stage_status) self.assertEqual(previous_activity, self.instance.last_activity_timestamp) - self.assertEqual( - previous_sync_activity, self.sync_session.last_activity_timestamp - ) + self.assertEqual(previous_sync_activity, self.sync_session.last_activity_timestamp) class TransferSessionAndStoreTestCase(TestCase): @@ -363,11 +343,7 @@ def test_get_touched_record_ids_for_model__class(self): def test_get_touched_record_ids_for_model__string(self): self.assertEqual( [self.user.id], - list( - self.instance.get_touched_record_ids_for_model( - MyUser.morango_model_name - ) - ), + list(self.instance.get_touched_record_ids_for_model(MyUser.morango_model_name)), ) @@ -384,7 +360,9 @@ def test_cached_clean_fields(self, mock_clean_fields): f = Facility(name="test") sync_filter = Filter("test") f.cached_clean_fields({}, exclude=["test1"], sync_filter=sync_filter) - mock_clean_fields.assert_called_once_with(exclude=["test1", "parent"], sync_filter=sync_filter) + mock_clean_fields.assert_called_once_with( + exclude=["test1", "parent"], sync_filter=sync_filter + ) @mock.patch("morango.models.core.SyncableModel.clean_fields") def test_deferred_clean_fields(self, mock_clean_fields): diff --git a/tests/testapp/tests/models/test_fsic_utils.py b/tests/testapp/tests/models/test_fsic_utils.py index 3b1f0c0d..2ebd73b9 100644 --- a/tests/testapp/tests/models/test_fsic_utils.py +++ b/tests/testapp/tests/models/test_fsic_utils.py @@ -1,10 +1,12 @@ -from morango.models.fsic_utils import chunk_fsic_v2 -from morango.models.fsic_utils import expand_fsic_for_use -from morango.models.fsic_utils import remove_redundant_instance_counters -from morango.models.fsic_utils import calculate_directional_fsic_diff_v2 - from django.test import TestCase +from morango.models.fsic_utils import ( + calculate_directional_fsic_diff_v2, + chunk_fsic_v2, + expand_fsic_for_use, + remove_redundant_instance_counters, +) + class TestFSICUtils(TestCase): def test_expand_fsic_for_use(self): diff --git a/tests/testapp/tests/sync/stream/test_core.py b/tests/testapp/tests/sync/stream/test_core.py index d858b1ef..cf12e8e0 100644 --- a/tests/testapp/tests/sync/stream/test_core.py +++ b/tests/testapp/tests/sync/stream/test_core.py @@ -1,12 +1,6 @@ from django.test import SimpleTestCase -from morango.sync.stream.core import Buffer -from morango.sync.stream.core import FlatMap -from morango.sync.stream.core import Pipeline -from morango.sync.stream.core import Sink -from morango.sync.stream.core import Source -from morango.sync.stream.core import Transform -from morango.sync.stream.core import Unbuffer +from morango.sync.stream.core import Buffer, FlatMap, Pipeline, Sink, Source, Transform, Unbuffer class FakeSource(Source): diff --git a/tests/testapp/tests/sync/stream/test_serialize.py b/tests/testapp/tests/sync/stream/test_serialize.py index 6b1be078..140714aa 100644 --- a/tests/testapp/tests/sync/stream/test_serialize.py +++ b/tests/testapp/tests/sync/stream/test_serialize.py @@ -5,16 +5,15 @@ from django.test import SimpleTestCase from morango.models.certificates import Filter -from morango.models.core import InstanceIDModel -from morango.models.core import RecordMaxCounter -from morango.models.core import Store -from morango.models.core import SyncableModel -from morango.sync.stream.serialize import AppModelSource -from morango.sync.stream.serialize import ModelPartitionBuffer -from morango.sync.stream.serialize import SerializeTask -from morango.sync.stream.serialize import StoreLookup -from morango.sync.stream.serialize import StoreUpdate -from morango.sync.stream.serialize import WriteSink +from morango.models.core import InstanceIDModel, RecordMaxCounter, Store, SyncableModel +from morango.sync.stream.serialize import ( + AppModelSource, + ModelPartitionBuffer, + SerializeTask, + StoreLookup, + StoreUpdate, + WriteSink, +) class SerializeTaskTestCase(SimpleTestCase): @@ -58,9 +57,7 @@ def test_prefix_conditions__with_filter(self): source = AppModelSource(profile="test", sync_filter=sync_filter) conditions = list(source.prefix_conditions()) self.assertEqual(len(conditions), 2) - self.assertEqual( - str(conditions[0]), "(AND: ('_morango_partition__startswith', 'a'))" - ) + self.assertEqual(str(conditions[0]), "(AND: ('_morango_partition__startswith', 'a'))") @mock.patch("morango.sync.stream.serialize.syncable_models.get_model_querysets") def test_stream__no_partition(self, mock_get_model_querysets): @@ -107,9 +104,7 @@ def test_stream__partition(self, mock_get_model_querysets): qs.filter.return_value = qs qs.iterator.return_value = [obj, obj] - source = AppModelSource( - profile="test", sync_filter=Filter("a"), dirty_only=False - ) + source = AppModelSource(profile="test", sync_filter=Filter("a"), dirty_only=False) tasks = list(source.stream()) self.assertEqual(len(tasks), 1) @@ -273,16 +268,12 @@ def _bulk_counter_create(objs, **kwargs): # Verify dirty bit update on app models model.syncing_objects.filter.assert_called_once_with(id__in=["obj_1", "obj_2"]) - model.syncing_objects.filter().update.assert_called_once_with( - update_dirty_bit_to=False - ) + model.syncing_objects.filter().update.assert_called_once_with(update_dirty_bit_to=False) @mock.patch("morango.sync.stream.serialize.RecordMaxCounter.objects.filter") @mock.patch("morango.sync.stream.serialize.RecordMaxCounter.objects.bulk_create") @mock.patch("morango.sync.stream.serialize.Store.objects.bulk_create") - def test_consume__create_fail( - self, mock_store_bulk, mock_counter_bulk, mock_counter_filter - ): + def test_consume__create_fail(self, mock_store_bulk, mock_counter_bulk, mock_counter_filter): model = mock.Mock() obj1 = mock.Mock(id="obj_1") task1 = SerializeTask(model, obj1) @@ -315,9 +306,7 @@ def test_consume__create_fail( # Verify dirty bit update on app models model.syncing_objects.filter.assert_called_once_with(id__in=["obj_1"]) - model.syncing_objects.filter().update.assert_called_once_with( - update_dirty_bit_to=False - ) + model.syncing_objects.filter().update.assert_called_once_with(update_dirty_bit_to=False) @mock.patch("morango.sync.stream.serialize.RecordMaxCounter.objects.filter") @mock.patch("morango.sync.stream.serialize.RecordMaxCounter.objects.bulk_create") @@ -344,15 +333,11 @@ def test_handle_deleted( mock_counter_filter.assert_called_once_with( instance_id=self.current_id.id, store_model_id__in=["del_1"] ) - mock_counter_filter().update.assert_called_once_with( - counter=self.current_id.counter - ) + mock_counter_filter().update.assert_called_once_with(counter=self.current_id.counter) mock_store_records.exclude.assert_called_once_with( recordmaxcounter__instance_id=self.current_id.id ) - mock_store_records.exclude().values_list.assert_called_once_with( - "id", flat=True - ) + mock_store_records.exclude().values_list.assert_called_once_with("id", flat=True) mock_rmc_bulk.assert_called_once() rmc = mock_rmc_bulk.call_args[0][0][0] self.assertEqual(rmc.store_model_id, "del_1") @@ -375,9 +360,7 @@ def test_handle_hard_deleted(self, mock_store_filter, mock_hard_deleted_filter): ) mock_hard_deleted_filter().delete.assert_called_once() - @mock.patch( - "morango.sync.stream.serialize.DatabaseMaxCounter.objects.update_or_create" - ) + @mock.patch("morango.sync.stream.serialize.DatabaseMaxCounter.objects.update_or_create") def test_update_counters(self, mock_update_or_create): self.sink.sync_filter = Filter("a") self.sink._update_counters() @@ -388,9 +371,7 @@ def test_update_counters(self, mock_update_or_create): defaults={"counter": self.current_id.counter}, ) - @mock.patch( - "morango.sync.stream.serialize.DatabaseMaxCounter.objects.update_or_create" - ) + @mock.patch("morango.sync.stream.serialize.DatabaseMaxCounter.objects.update_or_create") def test_update_counters__no_filter(self, mock_update_or_create): self.sink._update_counters() self.assertEqual(mock_update_or_create.call_count, 1) diff --git a/tests/testapp/tests/sync/test_context.py b/tests/testapp/tests/sync/test_context.py index 7ab6c960..d01b8bda 100644 --- a/tests/testapp/tests/sync/test_context.py +++ b/tests/testapp/tests/sync/test_context.py @@ -1,23 +1,22 @@ import pickle import mock -from django.test import SimpleTestCase -from django.test import TestCase +from django.test import SimpleTestCase, TestCase -from ..helpers import create_dummy_store_data -from ..helpers import TestSessionContext -from morango.constants import transfer_stages -from morango.constants import transfer_statuses +from morango.constants import transfer_stages, transfer_statuses from morango.errors import MorangoContextUpdateError from morango.models.certificates import Filter -from morango.models.core import SyncSession -from morango.models.core import TransferSession -from morango.sync.context import CompositeSessionContext -from morango.sync.context import LocalSessionContext -from morango.sync.context import NetworkSessionContext -from morango.sync.context import SessionContext +from morango.models.core import SyncSession, TransferSession +from morango.sync.context import ( + CompositeSessionContext, + LocalSessionContext, + NetworkSessionContext, + SessionContext, +) from morango.sync.controller import SessionController +from ..helpers import TestSessionContext, create_dummy_store_data + class SessionContextTestCase(SimpleTestCase): def test_init__nothing(self): @@ -41,7 +40,9 @@ def test_init__no_transfer_session(self): sync_session = mock.Mock(spec=SyncSession) sync_filter = Filter("a") - context = TestSessionContext(sync_session=sync_session, sync_filter=sync_filter, is_push=True) + context = TestSessionContext( + sync_session=sync_session, sync_filter=sync_filter, is_push=True + ) self.assertEqual(sync_session, context.sync_session) self.assertEqual(sync_filter, context.filter) self.assertTrue(context.is_push) @@ -159,7 +160,7 @@ def test_update__basic(self): is_push=True, stage=transfer_stages.TRANSFERRING, stage_status=transfer_statuses.STARTED, - capabilities={"testing"} + capabilities={"testing"}, ) self.assertEqual(sync_filter, context.filter) self.assertTrue(context.is_push) @@ -169,9 +170,7 @@ def test_update__basic(self): @mock.patch("morango.sync.context.CAPABILITIES", {"testing"}) def test_update__with_transfer_session(self): - context = TestSessionContext( - capabilities={"testing"} - ) + context = TestSessionContext(capabilities={"testing"}) sync_session = mock.Mock(spec=SyncSession) sync_filter = Filter("a") @@ -273,8 +272,9 @@ def test_local(self, mock_parse_capabilities): @mock.patch("morango.sync.context.parse_capabilities_from_server_request") def test_network(self, mock_parse_capabilities): - conn = mock.Mock(spec="morango.sync.syncsession.NetworkSyncConnection", - server_info=dict(capabilities={})) + conn = mock.Mock( + spec="morango.sync.syncsession.NetworkSyncConnection", server_info=dict(capabilities={}) + ) mock_parse_capabilities.return_value = {} context = NetworkSessionContext(conn) @@ -292,8 +292,9 @@ def test_composite(self): request = mock.Mock(spec="django.http.request.HttpRequest") local = LocalSessionContext(request=request) - conn = mock.Mock(spec="morango.sync.syncsession.NetworkSyncConnection", - server_info=dict(capabilities={})) + conn = mock.Mock( + spec="morango.sync.syncsession.NetworkSyncConnection", server_info=dict(capabilities={}) + ) network = NetworkSessionContext(conn) composite = CompositeSessionContext([local, network]) @@ -340,8 +341,12 @@ def test_state_behavior(self): for stage in self.stages: self.context.update(stage=stage, stage_status=transfer_statuses.PENDING) - self.sub_context_a.update_state.assert_called_with(stage=stage, stage_status=transfer_statuses.PENDING) - self.sub_context_b.update_state.assert_called_with(stage=stage, stage_status=transfer_statuses.PENDING) + self.sub_context_a.update_state.assert_called_with( + stage=stage, stage_status=transfer_statuses.PENDING + ) + self.sub_context_b.update_state.assert_called_with( + stage=stage, stage_status=transfer_statuses.PENDING + ) self.sub_context_a.update_state.reset_mock() self.sub_context_b.update_state.reset_mock() @@ -371,8 +376,12 @@ def test_state_behavior(self): self.assertIs(prepared_context, self.sub_context_b) self.context.update(stage_status=transfer_statuses.COMPLETED) - self.sub_context_a.update_state.assert_called_once_with(stage=None, stage_status=transfer_statuses.COMPLETED) - self.sub_context_b.update_state.assert_called_once_with(stage=None, stage_status=transfer_statuses.COMPLETED) + self.sub_context_a.update_state.assert_called_once_with( + stage=None, stage_status=transfer_statuses.COMPLETED + ) + self.sub_context_b.update_state.assert_called_once_with( + stage=None, stage_status=transfer_statuses.COMPLETED + ) self.sub_context_a.update_state.reset_mock() self.sub_context_b.update_state.reset_mock() @@ -402,7 +411,7 @@ def _middleware(context): context.transfer_session = TransferSession( sync_session=SyncSession(), transfer_stage=transfer_stages.INITIALIZING, - transfer_stage_status=transfer_statuses.PENDING + transfer_stage_status=transfer_statuses.PENDING, ) return transfer_statuses.COMPLETED @@ -427,7 +436,7 @@ def _middleware(context): context.transfer_session = TransferSession( sync_session=SyncSession(), transfer_stage=transfer_stages.DESERIALIZING, - transfer_stage_status=transfer_statuses.PENDING + transfer_stage_status=transfer_statuses.PENDING, ) return transfer_statuses.COMPLETED diff --git a/tests/testapp/tests/sync/test_controller.py b/tests/testapp/tests/sync/test_controller.py index 1b302e75..2425df64 100644 --- a/tests/testapp/tests/sync/test_controller.py +++ b/tests/testapp/tests/sync/test_controller.py @@ -4,26 +4,16 @@ import factory import mock -from django.test import SimpleTestCase -from django.test import TestCase -from facility_profile.models import Facility -from facility_profile.models import InteractionLog -from facility_profile.models import MyUser -from facility_profile.models import SummaryLog +from django.test import SimpleTestCase, TestCase +from facility_profile.models import Facility, InteractionLog, MyUser, SummaryLog -from ..compat import EnvironmentVarGuard -from ..helpers import FacilityModelFactory -from ..helpers import serialized_facility_factory -from ..helpers import TestSessionContext -from morango.constants import transfer_stages -from morango.constants import transfer_statuses +from morango.constants import transfer_stages, transfer_statuses from morango.models.certificates import Filter -from morango.models.core import DeletedModels -from morango.models.core import InstanceIDModel -from morango.models.core import RecordMaxCounter -from morango.models.core import Store -from morango.sync.controller import MorangoProfileController -from morango.sync.controller import SessionController +from morango.models.core import DeletedModels, InstanceIDModel, RecordMaxCounter, Store +from morango.sync.controller import MorangoProfileController, SessionController + +from ..compat import EnvironmentVarGuard +from ..helpers import FacilityModelFactory, TestSessionContext, serialized_facility_factory class StoreModelFacilityFactory(factory.django.DjangoModelFactory): @@ -52,10 +42,7 @@ def test_all_models_get_serialized(self): def test_no_models_get_serialized(self): # set dirty bit off on new models created - [ - FacilityModelFactory.build().save(update_dirty_bit_to=False) - for _ in range(self.range) - ] + [FacilityModelFactory.build().save(update_dirty_bit_to=False) for _ in range(self.range)] # only models with dirty bit on should be serialized self.mc.serialize_into_store() self.assertFalse(Store.objects.exists()) @@ -298,9 +285,7 @@ def test_new_rmc_for_existing_model(self): Facility.objects.update(name="facility") self.mc.serialize_into_store() - new_rmc = RecordMaxCounter.objects.get( - instance_id=new_id.id, store_model_id=self.fac1.id - ) + new_rmc = RecordMaxCounter.objects.get(instance_id=new_id.id, store_model_id=self.fac1.id) new_store_record = Store.objects.get(id=self.fac1.id) self.assertEqual(new_rmc.counter, new_store_record.last_saved_counter) @@ -335,9 +320,7 @@ def test_new_rmc_for_non_existent_model(self): new_fac = FacilityModelFactory(name="college") self.mc.serialize_into_store() - new_rmc = RecordMaxCounter.objects.get( - instance_id=new_id.id, store_model_id=new_fac.id - ) + new_rmc = RecordMaxCounter.objects.get(instance_id=new_id.id, store_model_id=new_fac.id) new_store_record = Store.objects.get(id=new_fac.id) self.assertNotEqual(new_id.id, self.current_id.id) @@ -377,9 +360,7 @@ def test_deleted_models_deletes_them_in_app(self): self.mc.deserialize_from_store() # deleted flag on store should delete model in app layer - Store.objects.update_or_create( - defaults={"deleted": True, "dirty_bit": True}, id=self.ident - ) + Store.objects.update_or_create(defaults={"deleted": True, "dirty_bit": True}, id=self.ident) self.mc.deserialize_from_store() self.assertFalse(Facility.objects.filter(id=self.ident).exists()) @@ -422,9 +403,7 @@ def test_record_with_dirty_bit_off_doesnt_deserialize(self): def test_broken_fk_leaves_store_dirty_bit(self): log_id = uuid.uuid4().hex - serialized = json.dumps( - {"user_id": "40de9a3fded95d7198f200c78e559353", "id": log_id} - ) + serialized = json.dumps({"user_id": "40de9a3fded95d7198f200c78e559353", "id": log_id}) st = StoreModelFacilityFactory( id=log_id, serialized=serialized, model_name="contentsummarylog" ) @@ -585,14 +564,10 @@ def test_models_created_successfully(self): self.assertEqual(child2[0].parent_id, root.id) def test_deserialization_of_model_with_missing_parent(self): - self._test_deserialization_of_model_with_missing_parent( - correct_self_ref_fk=True - ) + self._test_deserialization_of_model_with_missing_parent(correct_self_ref_fk=True) def test_deserialization_of_model_with_mismatched_self_ref_fk(self): - self._test_deserialization_of_model_with_missing_parent( - correct_self_ref_fk=False - ) + self._test_deserialization_of_model_with_missing_parent(correct_self_ref_fk=False) def _test_deserialization_of_model_with_missing_parent(self, correct_self_ref_fk): root = FacilityModelFactory() @@ -709,19 +684,13 @@ def test_deserialization_of_model_with_valid_foreignkey_referent(self): class SessionControllerTestCase(SimpleTestCase): def setUp(self): super(SessionControllerTestCase, self).setUp() - self.middleware = [ - mock.Mock(related_stage=stage) for stage, _ in transfer_stages.CHOICES - ] + self.middleware = [mock.Mock(related_stage=stage) for stage, _ in transfer_stages.CHOICES] self.context = TestSessionContext() - self.controller = SessionController.build( - middleware=self.middleware, context=self.context - ) + self.controller = SessionController.build(middleware=self.middleware, context=self.context) @contextlib.contextmanager def _mock_method(self, method): - with mock.patch( - "morango.sync.controller.SessionController.{}".format(method) - ) as invoke: + with mock.patch("morango.sync.controller.SessionController.{}".format(method)) as invoke: yield invoke invoke.reset_mock() @@ -847,9 +816,7 @@ def test_invoke_middleware(self): middleware = self.middleware[0] middleware.return_value = transfer_statuses.STARTED - with mock.patch.object( - TestSessionContext, "update_state", wraps=context.update_state - ) as m: + with mock.patch.object(TestSessionContext, "update_state", wraps=context.update_state) as m: result = self.controller._invoke_middleware(context, middleware) self.assertEqual(result, transfer_statuses.STARTED) diff --git a/tests/testapp/tests/sync/test_db.py b/tests/testapp/tests/sync/test_db.py index 0bee0403..f6503e40 100644 --- a/tests/testapp/tests/sync/test_db.py +++ b/tests/testapp/tests/sync/test_db.py @@ -5,25 +5,22 @@ import pytest from django.conf import settings from django.db import connection -from django.test import override_settings -from django.test import TransactionTestCase +from django.test import TransactionTestCase, override_settings from django.utils import timezone -from ..helpers import create_buffer_and_store_dummy_data from morango.models.certificates import Filter -from morango.models.core import Store -from morango.models.core import SyncSession -from morango.models.core import TransferSession +from morango.models.core import Store, SyncSession, TransferSession from morango.sync.backends.utils import load_backend from morango.sync.db import begin_transaction +from ..helpers import create_buffer_and_store_dummy_data DBBackend = load_backend(connection) def _concurrent_store_write(thread_event, store_id): while not thread_event.is_set(): - sleep(.1) + sleep(0.1) Store.objects.filter(id=store_id).delete() connection.close() @@ -92,17 +89,23 @@ def test_transaction_isolation_handling(self): # this test is only for postgres, but we don't want the code to know it's a test with override_settings(MORANGO_TEST_POSTGRESQL=False): try: - self.assertNotEqual(connection.connection.isolation_level, ISOLATION_LEVEL_REPEATABLE_READ) + self.assertNotEqual( + connection.connection.isolation_level, ISOLATION_LEVEL_REPEATABLE_READ + ) with begin_transaction(Filter(store.partition), isolated=True): - self.assertEqual(connection.connection.isolation_level, ISOLATION_LEVEL_REPEATABLE_READ) + self.assertEqual( + connection.connection.isolation_level, ISOLATION_LEVEL_REPEATABLE_READ + ) s = Store.objects.get(id=store.id) concurrent_event.set() - sleep(.2) + sleep(0.2) s.last_saved_counter += 1 s.save() raise AssertionError("Didn't raise transactional error") except Exception as e: self.assertTrue(DBBackend._is_transaction_isolation_error(e)) - self.assertNotEqual(connection.connection.isolation_level, ISOLATION_LEVEL_REPEATABLE_READ) + self.assertNotEqual( + connection.connection.isolation_level, ISOLATION_LEVEL_REPEATABLE_READ + ) finally: concurrent_thread.join(5) diff --git a/tests/testapp/tests/sync/test_operations.py b/tests/testapp/tests/sync/test_operations.py index 2a41bed6..4c86c980 100644 --- a/tests/testapp/tests/sync/test_operations.py +++ b/tests/testapp/tests/sync/test_operations.py @@ -3,46 +3,45 @@ import mock from django.db import connection -from django.test import override_settings -from django.test import TestCase +from django.test import TestCase, override_settings from django.utils import timezone -from facility_profile.models import ConditionalLog -from facility_profile.models import Facility -from facility_profile.models import MyUser -from facility_profile.models import SummaryLog +from facility_profile.models import ConditionalLog, Facility, MyUser, SummaryLog -from ..helpers import create_buffer_and_store_dummy_data -from ..helpers import create_dummy_store_data from morango.constants import transfer_statuses from morango.constants.capabilities import FSIC_V2_FORMAT from morango.errors import MorangoLimitExceeded from morango.models.certificates import Filter -from morango.models.core import Buffer -from morango.models.core import DatabaseIDModel -from morango.models.core import DatabaseMaxCounter -from morango.models.core import InstanceIDModel -from morango.models.core import RecordMaxCounter -from morango.models.core import RecordMaxCounterBuffer -from morango.models.core import Store -from morango.models.core import SyncSession -from morango.models.core import TransferSession +from morango.models.core import ( + Buffer, + DatabaseIDModel, + DatabaseMaxCounter, + InstanceIDModel, + RecordMaxCounter, + RecordMaxCounterBuffer, + Store, + SyncSession, + TransferSession, +) from morango.sync.backends.utils import load_backend from morango.sync.context import LocalSessionContext -from morango.sync.controller import MorangoProfileController -from morango.sync.controller import SessionController -from morango.sync.operations import _dequeue_into_store -from morango.sync.operations import _deserialize_from_store -from morango.sync.operations import _queue_into_buffer_v1 -from morango.sync.operations import _queue_into_buffer_v2 -from morango.sync.operations import CleanupOperation -from morango.sync.operations import InitializeOperation -from morango.sync.operations import ProducerDequeueOperation -from morango.sync.operations import ProducerQueueOperation -from morango.sync.operations import ReceiverDequeueOperation -from morango.sync.operations import ReceiverDeserializeOperation -from morango.sync.operations import ReceiverQueueOperation +from morango.sync.controller import MorangoProfileController, SessionController +from morango.sync.operations import ( + CleanupOperation, + InitializeOperation, + ProducerDequeueOperation, + ProducerQueueOperation, + ReceiverDequeueOperation, + ReceiverDeserializeOperation, + ReceiverQueueOperation, + _dequeue_into_store, + _deserialize_from_store, + _queue_into_buffer_v1, + _queue_into_buffer_v2, +) from morango.sync.syncsession import TransferClient +from ..helpers import create_buffer_and_store_dummy_data, create_dummy_store_data + DBBackend = load_backend(connection) @@ -215,9 +214,7 @@ def test_local_initialize_operation__server(self): operation = InitializeOperation() self.assertEqual(transfer_statuses.COMPLETED, operation.handle(self.context)) self.context.update.assert_called_once() - transfer_session = self.context.update.call_args_list[0][1].get( - "transfer_session" - ) + transfer_session = self.context.update.call_args_list[0][1].get("transfer_session") self.assertEqual(id, transfer_session.id) self.assertEqual(records_total, transfer_session.records_total) self.assertEqual(client_fsic, transfer_session.client_fsic) @@ -235,9 +232,7 @@ def test_local_initialize_operation__resume(self): self.context.transfer_session = None operation = InitializeOperation() self.assertEqual(transfer_statuses.COMPLETED, operation.handle(self.context)) - self.context.update.assert_called_once_with( - transfer_session=self.transfer_session - ) + self.context.update.assert_called_once_with(transfer_session=self.transfer_session) def test_local_queue_operation(self): fsics = {self.data["group1_id"].id: 1, self.data["group2_id"].id: 1} @@ -288,7 +283,10 @@ def test_very_many_instances_in_fsic(self): """ Regression test against 'Expression tree is too large (maximum depth 1000)' error with large fsics """ - fsics = {"super": {}, "sub": {"": {self.data["group1_id"].id: 1, self.data["group2_id"].id: 1}}} + fsics = { + "super": {}, + "sub": {"": {self.data["group1_id"].id: 1, self.data["group2_id"].id: 1}}, + } fsics["sub"][""].update({uuid.uuid4().hex: i for i in range(10000)}) self.transfer_session.client_fsic = json.dumps(fsics) self.transfer_session.server_fsic = json.dumps({"super": {}, "sub": {}}) @@ -303,7 +301,10 @@ def test_very_many_partitions_in_fsic(self): """ Regression test against 'Expression tree is too large (maximum depth 1000)' error with large fsics """ - fsics = {"super": {}, "sub": {"": {self.data["group1_id"].id: 1, self.data["group2_id"].id: 1}}} + fsics = { + "super": {}, + "sub": {"": {self.data["group1_id"].id: 1, self.data["group2_id"].id: 1}}, + } for i in range(10000): fsics["sub"][uuid.uuid4().hex] = {uuid.uuid4().hex: i} self.transfer_session.client_fsic = json.dumps(fsics) @@ -319,7 +320,10 @@ def test_very_many_partitions_and_instances_in_fsic(self): """ Regression test against 'Expression tree is too large (maximum depth 1000)' error with large fsics """ - fsics = {"super": {}, "sub": {"": {self.data["group1_id"].id: 1, self.data["group2_id"].id: 1}}} + fsics = { + "super": {}, + "sub": {"": {self.data["group1_id"].id: 1, self.data["group2_id"].id: 1}}, + } for i in range(99): fsics["sub"][uuid.uuid4().hex] = {uuid.uuid4().hex: i for i in range(999)} self.transfer_session.client_fsic = json.dumps(fsics) @@ -331,7 +335,10 @@ def test_very_many_partitions_and_instances_in_fsic(self): assertRecordsBuffered(self.data["group2_c1"]) def test_too_many_fsic_partitions(self): - fsics = {"super": {}, "sub": {"": {self.data["group1_id"].id: 1, self.data["group2_id"].id: 1}}} + fsics = { + "super": {}, + "sub": {"": {self.data["group1_id"].id: 1, self.data["group2_id"].id: 1}}, + } for i in range(5000): fsics["sub"][uuid.uuid4().hex] = {uuid.uuid4().hex: i for i in range(2)} self.transfer_session.client_fsic = json.dumps(fsics) @@ -340,7 +347,10 @@ def test_too_many_fsic_partitions(self): _queue_into_buffer_v2(self.transfer_session, chunk_size=10) def test_too_many_fsic_instances(self): - fsics = {"super": {}, "sub": {"": {self.data["group1_id"].id: 1, self.data["group2_id"].id: 1}}} + fsics = { + "super": {}, + "sub": {"": {self.data["group1_id"].id: 1, self.data["group2_id"].id: 1}}, + } for i in range(2): fsics["sub"][uuid.uuid4().hex] = {uuid.uuid4().hex: i for i in range(5000)} self.transfer_session.client_fsic = json.dumps(fsics) @@ -371,7 +381,10 @@ def test_fsic_counters(self): assertRecordsNotBuffered(self.data["group2_c1"]) def test_fsic_counters_too_high(self): - fsics = {"super": {}, "sub": {"": {self.data["group1_id"].id: 100, self.data["group2_id"].id: 100}}} + fsics = { + "super": {}, + "sub": {"": {self.data["group1_id"].id: 100, self.data["group2_id"].id: 100}}, + } self.transfer_session.client_fsic = json.dumps(fsics) self.transfer_session.server_fsic = json.dumps(fsics) _queue_into_buffer_v2(self.transfer_session) @@ -390,7 +403,10 @@ def test_valid_fsic_but_invalid_partition(self): assertRecordsNotBuffered([self.data["user4"]]) def test_local_queue_operation(self): - fsics = {"super": {}, "sub": {"": {self.data["group1_id"].id: 1, self.data["group2_id"].id: 1}}} + fsics = { + "super": {}, + "sub": {"": {self.data["group1_id"].id: 1, self.data["group2_id"].id: 1}}, + } self.transfer_session.client_fsic = json.dumps(fsics) self.transfer_session.server_fsic = json.dumps({"super": {}, "sub": {}}) self.assertEqual(0, self.transfer_session.records_total or 0) @@ -405,7 +421,10 @@ def test_local_queue_operation(self): @mock.patch("morango.sync.operations._queue_into_buffer_v2") def test_local_queue_operation__noop(self, mock_queue): - fsics = {"super": {}, "sub": {"": {self.data["group1_id"].id: 1, self.data["group2_id"].id: 1}}} + fsics = { + "super": {}, + "sub": {"": {self.data["group1_id"].id: 1, self.data["group2_id"].id: 1}}, + } self.transfer_session.client_fsic = json.dumps(fsics) self.transfer_session.server_fsic = json.dumps({"super": {}, "sub": {}}) @@ -418,9 +437,7 @@ def test_local_queue_operation__noop(self, mock_queue): mock_queue.assert_not_called() -@override_settings( - MORANGO_SERIALIZE_BEFORE_QUEUING=False, MORANGO_DISABLE_FSIC_V2_FORMAT=False -) +@override_settings(MORANGO_SERIALIZE_BEFORE_QUEUING=False, MORANGO_DISABLE_FSIC_V2_FORMAT=False) class FSICPartitionEdgeCaseQueuingTestCase(TestCase): def setUp(self): # instance IDs @@ -461,7 +478,9 @@ def clear_dmcs(self): def fsic_from_dmcs(self, filters, dmc_tuples): self.create_dmcs(dmc_tuples) - return DatabaseMaxCounter.calculate_filter_specific_instance_counters(filters, v2_format=True) + return DatabaseMaxCounter.calculate_filter_specific_instance_counters( + filters, v2_format=True + ) def initialize_sessions(self, filters): # create controllers for store/buffer operations @@ -474,13 +493,13 @@ def initialize_sessions(self, filters): profile="facilitydata", last_activity_timestamp=timezone.now(), ) - self.transfer_session = ( - self.sync_session.current_transfer_session - ) = TransferSession.objects.create( - id=uuid.uuid4().hex, - sync_session=self.sync_session, - push=True, - last_activity_timestamp=timezone.now(), + self.transfer_session = self.sync_session.current_transfer_session = ( + TransferSession.objects.create( + id=uuid.uuid4().hex, + sync_session=self.sync_session, + push=True, + last_activity_timestamp=timezone.now(), + ) ) self.transfer_session.filter = str(filters) @@ -623,11 +642,7 @@ def setUp(self): push=True, last_activity_timestamp=timezone.now(), ) - self.data.update( - create_buffer_and_store_dummy_data( - self.transfer_session.id - ) - ) + self.data.update(create_buffer_and_store_dummy_data(self.transfer_session.id)) self.context = mock.Mock( spec=LocalSessionContext, transfer_session=self.transfer_session, @@ -645,27 +660,20 @@ def assert_store_records_not_tagged_with_last_session(self, store_ids): session_id = self.transfer_session.id for store_id in store_ids: try: - assert ( - Store.objects.get(id=store_id).last_transfer_session_id - != session_id - ) + assert Store.objects.get(id=store_id).last_transfer_session_id != session_id except Store.DoesNotExist: pass def test_dequeuing_sets_last_session(self): - store_ids = [ - self.data[key] for key in ["model2", "model3", "model4", "model5", "model7"] - ] + store_ids = [self.data[key] for key in ["model2", "model3", "model4", "model5", "model7"]] self.assert_store_records_not_tagged_with_last_session(store_ids) - _dequeue_into_store(self.transfer_session, self.transfer_session.client_fsic, v2_format=False) + _dequeue_into_store( + self.transfer_session, self.transfer_session.client_fsic, v2_format=False + ) # this one is a reverse fast forward, so it doesn't modify the store record and shouldn't be tagged self.assert_store_records_not_tagged_with_last_session([self.data["model1"]]) self.assert_store_records_tagged_with_last_session(store_ids) - tagged_actual = set( - self.transfer_session.get_touched_record_ids_for_model( - "facility" - ) - ) + tagged_actual = set(self.transfer_session.get_touched_record_ids_for_model("facility")) tagged_expected = set(store_ids) assert tagged_actual == tagged_expected @@ -695,9 +703,7 @@ def test_dequeuing_delete_rmcb_records(self): def test_dequeuing_delete_buffered_records(self): self.assertTrue(Buffer.objects.filter(model_uuid=self.data["model1"]).exists()) with connection.cursor() as cursor: - DBBackend._dequeuing_delete_buffered_records( - cursor, self.transfer_session.id - ) + DBBackend._dequeuing_delete_buffered_records(cursor, self.transfer_session.id) self.assertFalse(Buffer.objects.filter(model_uuid=self.data["model1"]).exists()) # ensure other records were not deleted self.assertTrue(Buffer.objects.filter(model_uuid=self.data["model2"]).exists()) @@ -752,9 +758,7 @@ def test_dequeuing_merge_conflict_buffer_rmcb_greater_than_rmc(self): self.assertFalse(store.deleted) with connection.cursor() as cursor: current_id = InstanceIDModel.get_current_instance_and_increment_counter() - DBBackend._dequeuing_merge_conflict_buffer( - cursor, current_id, self.transfer_session.id - ) + DBBackend._dequeuing_merge_conflict_buffer(cursor, current_id, self.transfer_session.id) store = Store.objects.get(id=self.data["model2"]) self.assertEqual(store.last_saved_instance, current_id.id) self.assertEqual(store.last_saved_counter, current_id.counter) @@ -767,9 +771,7 @@ def test_dequeuing_merge_conflict_buffer_rmcb_less_rmc(self): self.assertEqual(store.conflicting_serialized_data, "store") with connection.cursor() as cursor: current_id = InstanceIDModel.get_current_instance_and_increment_counter() - DBBackend._dequeuing_merge_conflict_buffer( - cursor, current_id, self.transfer_session.id - ) + DBBackend._dequeuing_merge_conflict_buffer(cursor, current_id, self.transfer_session.id) store = Store.objects.get(id=self.data["model5"]) self.assertEqual(store.last_saved_instance, current_id.id) self.assertEqual(store.last_saved_counter, current_id.counter) @@ -781,25 +783,19 @@ def test_dequeuing_merge_conflict_hard_delete(self): self.assertEqual(store.conflicting_serialized_data, "store") with connection.cursor() as cursor: current_id = InstanceIDModel.get_current_instance_and_increment_counter() - DBBackend._dequeuing_merge_conflict_buffer( - cursor, current_id, self.transfer_session.id - ) + DBBackend._dequeuing_merge_conflict_buffer(cursor, current_id, self.transfer_session.id) store.refresh_from_db() self.assertEqual(store.serialized, "") self.assertEqual(store.conflicting_serialized_data, "") def test_dequeuing_update_rmcs_last_saved_by(self): - self.assertFalse( - RecordMaxCounter.objects.filter(instance_id=self.current_id.id).exists() - ) + self.assertFalse(RecordMaxCounter.objects.filter(instance_id=self.current_id.id).exists()) with connection.cursor() as cursor: current_id = InstanceIDModel.get_current_instance_and_increment_counter() DBBackend._dequeuing_update_rmcs_last_saved_by( cursor, current_id, self.transfer_session.id ) - self.assertTrue( - RecordMaxCounter.objects.filter(instance_id=current_id.id).exists() - ) + self.assertTrue(RecordMaxCounter.objects.filter(instance_id=current_id.id).exists()) def test_dequeuing_delete_mc_buffer(self): self.assertTrue(Buffer.objects.filter(model_uuid=self.data["model2"]).exists()) @@ -839,14 +835,10 @@ def test_dequeuing_delete_mc_rmcb(self): ) def test_dequeuing_insert_remaining_buffer(self): - self.assertNotEqual( - Store.objects.get(id=self.data["model3"]).serialized, "buffer" - ) + self.assertNotEqual(Store.objects.get(id=self.data["model3"]).serialized, "buffer") self.assertFalse(Store.objects.filter(id=self.data["model4"]).exists()) with connection.cursor() as cursor: - DBBackend._dequeuing_insert_remaining_buffer( - cursor, self.transfer_session.id - ) + DBBackend._dequeuing_insert_remaining_buffer(cursor, self.transfer_session.id) self.assertEqual(Store.objects.get(id=self.data["model3"]).serialized, "buffer") self.assertTrue(Store.objects.filter(id=self.data["model4"]).exists()) @@ -858,9 +850,7 @@ def test_dequeuing_insert_remaining_rmcb(self): ).exists() ) with connection.cursor() as cursor: - DBBackend._dequeuing_insert_remaining_buffer( - cursor, self.transfer_session.id - ) + DBBackend._dequeuing_insert_remaining_buffer(cursor, self.transfer_session.id) DBBackend._dequeuing_insert_remaining_rmcb(cursor, self.transfer_session.id) for i in self.data["model4_rmcb_ids"]: self.assertTrue( @@ -888,19 +878,17 @@ def test_dequeuing_delete_remaining_buffer(self): Buffer.objects.filter(transfer_session_id=self.transfer_session.id).exists() ) with connection.cursor() as cursor: - DBBackend._dequeuing_delete_remaining_buffer( - cursor, self.transfer_session.id - ) + DBBackend._dequeuing_delete_remaining_buffer(cursor, self.transfer_session.id) self.assertFalse( Buffer.objects.filter(transfer_session_id=self.transfer_session.id).exists() ) def test_dequeue_into_store(self): - _dequeue_into_store(self.transfer_session, self.transfer_session.client_fsic, v2_format=False) - # ensure a record with different transfer session id is not affected - self.assertTrue( - Buffer.objects.filter(transfer_session_id=self.data["tfs_id"]).exists() + _dequeue_into_store( + self.transfer_session, self.transfer_session.client_fsic, v2_format=False ) + # ensure a record with different transfer session id is not affected + self.assertTrue(Buffer.objects.filter(transfer_session_id=self.data["tfs_id"]).exists()) self.assertFalse(Store.objects.filter(id=self.data["model6"]).exists()) self.assertFalse( RecordMaxCounter.objects.filter( @@ -910,13 +898,9 @@ def test_dequeue_into_store(self): ) # ensure reverse fast forward records are not modified - self.assertNotEqual( - Store.objects.get(id=self.data["model1"]).serialized, "buffer" - ) + self.assertNotEqual(Store.objects.get(id=self.data["model1"]).serialized, "buffer") self.assertFalse( - RecordMaxCounter.objects.filter( - instance_id=self.data["model1_rmcb_ids"][1] - ).exists() + RecordMaxCounter.objects.filter(instance_id=self.data["model1_rmcb_ids"][1]).exists() ) # ensure records with merge conflicts are modified @@ -929,14 +913,10 @@ def test_dequeue_into_store(self): "buffer\nstore", ) self.assertTrue( - RecordMaxCounter.objects.filter( - instance_id=self.data["model2_rmcb_ids"][1] - ).exists() + RecordMaxCounter.objects.filter(instance_id=self.data["model2_rmcb_ids"][1]).exists() ) self.assertTrue( - RecordMaxCounter.objects.filter( - instance_id=self.data["model5_rmcb_ids"][1] - ).exists() + RecordMaxCounter.objects.filter(instance_id=self.data["model5_rmcb_ids"][1]).exists() ) self.assertEqual( Store.objects.get(id=self.data["model2"]).last_saved_instance, @@ -952,9 +932,7 @@ def test_dequeue_into_store(self): Store.objects.get(id=self.data["model3"]).serialized, "buffer" ) # serialized field is overwritten self.assertTrue( - RecordMaxCounter.objects.filter( - instance_id=self.data["model3_rmcb_ids"][1] - ).exists() + RecordMaxCounter.objects.filter(instance_id=self.data["model3_rmcb_ids"][1]).exists() ) self.assertEqual( Store.objects.get(id=self.data["model3"]).last_saved_instance, @@ -1033,14 +1011,13 @@ def test_local_cleanup(self): class DeserializationTestCases(TestCase): - def setUp(self): self.profile = "facilitydata" self.serialized_facility = { "id": uuid.uuid4().hex, "name": "test facility", - "now_date": timezone.now().isoformat() + "now_date": timezone.now().isoformat(), } self.serialized_user = { "id": uuid.uuid4().hex, @@ -1064,12 +1041,10 @@ def setUp(self): "content_id": uuid.uuid4().hex, } - def serialize_to_store(self, Model, data, post_serialization=None): + def serialize_to_store(self, Model, data): instance = Model(**data) instance.calculate_uuid() serialized = instance.serialize() - if post_serialization: - serialized.update(post_serialization) Store.objects.create( id=serialized["id"], serialized=json.dumps(serialized), @@ -1082,12 +1057,11 @@ def serialize_to_store(self, Model, data, post_serialization=None): model_name=instance.morango_model_name, ) - def serialize_all_to_store(self, post_serialization=None): - post_serialization = post_serialization or {} + def serialize_all_to_store(self): self.serialize_to_store(Facility, self.serialized_facility) - self.serialize_to_store(MyUser, self.serialized_user, post_serialization=post_serialization.get("user", {})) - self.serialize_to_store(SummaryLog, self.serialized_log1, post_serialization=post_serialization.get("log1", {})) - self.serialize_to_store(SummaryLog, self.serialized_log2, post_serialization=post_serialization.get("log2", {})) + self.serialize_to_store(MyUser, self.serialized_user) + self.serialize_to_store(SummaryLog, self.serialized_log1) + self.serialize_to_store(SummaryLog, self.serialized_log2) self.serialize_to_store(ConditionalLog, self.serialized_conditional) def assert_deserialization( @@ -1101,52 +1075,66 @@ def assert_deserialization( self.assertEqual( Facility.objects.filter(id=self.serialized_facility["id"]).exists(), facility_deserialized, - msg="Facility was not deserialized" if facility_deserialized else "Facility was deserialized" + msg="Facility was not deserialized" + if facility_deserialized + else "Facility was deserialized", ) self.assertEqual( MyUser.objects.filter(id=self.serialized_user["id"]).exists(), user_deserialized, - msg="User was not deserialized" if user_deserialized else "User was deserialized" + msg="User was not deserialized" if user_deserialized else "User was deserialized", ) self.assertEqual( SummaryLog.objects.filter(id=self.serialized_log1["id"]).exists(), log1_deserialized, - msg="Log1 was not deserialized" if log1_deserialized else "Log1 was deserialized" + msg="Log1 was not deserialized" if log1_deserialized else "Log1 was deserialized", ) self.assertEqual( SummaryLog.objects.filter(id=self.serialized_log2["id"]).exists(), log2_deserialized, - msg="Log2 was not deserialized" if log2_deserialized else "Log2 was deserialized" + msg="Log2 was not deserialized" if log2_deserialized else "Log2 was deserialized", ) self.assertEqual( ConditionalLog.objects.filter(id=self.serialized_conditional["id"]).exists(), conditional_deserialized, - msg="Conditional was not deserialized" if conditional_deserialized else "Conditional was deserialized" + msg="Conditional was not deserialized" + if conditional_deserialized + else "Conditional was deserialized", ) self.assertEqual( Store.objects.get(id=self.serialized_facility["id"]).dirty_bit, (not facility_deserialized), - msg="Facility store does not reflect deserialization" if not facility_deserialized else "Facility store reflects deserialization" + msg="Facility store does not reflect deserialization" + if not facility_deserialized + else "Facility store reflects deserialization", ) self.assertEqual( Store.objects.get(id=self.serialized_user["id"]).dirty_bit, (not user_deserialized), - msg="User store does not reflect deserialization" if not user_deserialized else "User store reflects deserialization" + msg="User store does not reflect deserialization" + if not user_deserialized + else "User store reflects deserialization", ) self.assertEqual( Store.objects.get(id=self.serialized_log1["id"]).dirty_bit, (not log1_deserialized), - msg="Log1 store does not reflect deserialization" if not log1_deserialized else "Log1 store reflects deserialization" + msg="Log1 store does not reflect deserialization" + if not log1_deserialized + else "Log1 store reflects deserialization", ) self.assertEqual( Store.objects.get(id=self.serialized_log2["id"]).dirty_bit, (not log2_deserialized), - msg="Log2 store does not reflect deserialization" if not log2_deserialized else "Log2 store reflects deserialization" + msg="Log2 store does not reflect deserialization" + if not log2_deserialized + else "Log2 store reflects deserialization", ) self.assertEqual( Store.objects.get(id=self.serialized_conditional["id"]).dirty_bit, (not conditional_deserialized), - msg="Conditional store does not reflect deserialization" if not conditional_deserialized else "Conditional store reflects deserialization" + msg="Conditional store does not reflect deserialization" + if not conditional_deserialized + else "Conditional store reflects deserialization", ) def test_successful_deserialization(self): @@ -1169,7 +1157,7 @@ def test_deserialization_with_missing_username(self): user_deserialized=False, log1_deserialized=False, log2_deserialized=False, - conditional_deserialized=False + conditional_deserialized=False, ) def test_deserialization_with_excessively_long_username(self): @@ -1184,18 +1172,20 @@ def test_deserialization_with_excessively_long_username(self): user_deserialized=False, log1_deserialized=False, log2_deserialized=False, - conditional_deserialized=False + conditional_deserialized=False, ) def test_deserialization_with_invalid_content_id(self): - self.serialize_all_to_store({"log1": {"content_id": "invalid"}}) + self.serialized_log1["content_id"] = "invalid" + + self.serialize_all_to_store() _deserialize_from_store(self.profile) self.assert_deserialization(log1_deserialized=False) - def test_deserialization_with_log_non_existent_user_id(self): + def test_deserialization_with_invalid_log_user_id(self): self.serialized_log1["user_id"] = uuid.uuid4().hex diff --git a/tests/testapp/tests/sync/test_session.py b/tests/testapp/tests/sync/test_session.py index 25c5baf7..1dffdc16 100644 --- a/tests/testapp/tests/sync/test_session.py +++ b/tests/testapp/tests/sync/test_session.py @@ -1,11 +1,8 @@ import mock from django.test import TestCase -from requests.exceptions import HTTPError -from requests.exceptions import RequestException +from requests.exceptions import HTTPError, RequestException - -from morango.sync.session import _length_of_headers -from morango.sync.session import SessionWrapper +from morango.sync.session import SessionWrapper, _length_of_headers class SessionWrapperTestCase(TestCase): @@ -25,16 +22,21 @@ def test_request(self, mocked_super_request): self.assertEqual(wrapper.bytes_received, 1024 + head_length) def test_request_user_agent(self): - from morango import __version__ as morango_version from requests import __version__ as requests_version + from morango import __version__ as morango_version + wrapper = SessionWrapper() - expected_user_agent = "morango/{} python-requests/{}".format(morango_version, requests_version) + expected_user_agent = "morango/{} python-requests/{}".format( + morango_version, requests_version + ) self.assertEqual(wrapper.headers["User-Agent"], expected_user_agent) with self.settings(CUSTOM_INSTANCE_INFO={"kolibri": "0.16.0"}): wrapper = SessionWrapper() - expected_user_agent = "morango/{} kolibri/0.16.0 python-requests/{}".format(morango_version, requests_version) + expected_user_agent = "morango/{} kolibri/0.16.0 python-requests/{}".format( + morango_version, requests_version + ) self.assertEqual(wrapper.headers["User-Agent"], expected_user_agent) @mock.patch("morango.sync.session.logger") @@ -55,9 +57,7 @@ def test_request__not_ok(self, mocked_super_request, mocked_logger): wrapper.request("GET", "test_url", is_test=True) mocked_super_request.assert_called_once_with("GET", "test_url", is_test=True) - mocked_logger.error.assert_called_once_with( - "HTTPError Reason: Connection timeout" - ) + mocked_logger.error.assert_called_once_with("HTTPError Reason: Connection timeout") @mock.patch("morango.sync.session.logger") @mock.patch("morango.sync.session.Session.request") @@ -70,9 +70,7 @@ def test_request__really_not_ok(self, mocked_super_request, mocked_logger): wrapper.request("GET", "test_url", is_test=True) mocked_super_request.assert_called_once_with("GET", "test_url", is_test=True) - mocked_logger.error.assert_called_once_with( - "RequestException Reason: (no response)" - ) + mocked_logger.error.assert_called_once_with("RequestException Reason: (no response)") @mock.patch("morango.sync.session.Session.prepare_request") def test_prepare_request(self, mocked_super_prepare_request): @@ -87,7 +85,5 @@ def test_prepare_request(self, mocked_super_prepare_request): mocked_super_prepare_request.assert_called_once_with(request) self.assertEqual(expected, actual) - head_length = len("GET /path/to/resource HTTP/1.1") + _length_of_headers( - headers - ) + head_length = len("GET /path/to/resource HTTP/1.1") + _length_of_headers(headers) self.assertEqual(wrapper.bytes_sent, 256 + head_length) diff --git a/tests/testapp/tests/sync/test_syncsession.py b/tests/testapp/tests/sync/test_syncsession.py index d41f5d87..120a90d9 100644 --- a/tests/testapp/tests/sync/test_syncsession.py +++ b/tests/testapp/tests/sync/test_syncsession.py @@ -6,31 +6,30 @@ from django.test.utils import override_settings from requests.exceptions import HTTPError -from ..helpers import BaseClientTestCase -from ..helpers import BaseTransferClientTestCase from morango.api.serializers import CertificateSerializer -from morango.constants import transfer_stages -from morango.constants import transfer_statuses +from morango.constants import transfer_stages, transfer_statuses from morango.constants.capabilities import ALLOW_CERTIFICATE_PUSHING -from morango.errors import CertificateSignatureInvalid -from morango.errors import MorangoError -from morango.errors import MorangoResumeSyncError -from morango.errors import MorangoServerDoesNotAllowNewCertPush -from morango.models.certificates import Certificate -from morango.models.certificates import Filter -from morango.models.certificates import Key -from morango.models.certificates import ScopeDefinition +from morango.errors import ( + CertificateSignatureInvalid, + MorangoError, + MorangoResumeSyncError, + MorangoServerDoesNotAllowNewCertPush, +) +from morango.models.certificates import Certificate, Filter, Key, ScopeDefinition from morango.models.core import SyncSession from morango.models.fields.crypto import SharedKey -from morango.sync.context import LocalSessionContext -from morango.sync.context import NetworkSessionContext +from morango.sync.context import LocalSessionContext, NetworkSessionContext from morango.sync.controller import MorangoProfileController from morango.sync.session import SessionWrapper -from morango.sync.syncsession import NetworkSyncConnection -from morango.sync.syncsession import PullClient -from morango.sync.syncsession import PushClient -from morango.sync.syncsession import SyncSessionClient -from morango.sync.syncsession import TransferClient +from morango.sync.syncsession import ( + NetworkSyncConnection, + PullClient, + PushClient, + SyncSessionClient, + TransferClient, +) + +from ..helpers import BaseClientTestCase, BaseTransferClientTestCase def mock_patch_decorator(func): @@ -95,17 +94,13 @@ def setUp(self): profile=self.profile, scope_definition=self.subset_scope_def, scope_version=self.subset_scope_def.version, - scope_params=json.dumps( - {"mainpartition": self.root_cert.id, "subpartition": "other"} - ), + scope_params=json.dumps({"mainpartition": self.root_cert.id, "subpartition": "other"}), public_key=Key(), ) self.root_cert.sign_certificate(self.unsaved_cert) self.controller = MorangoProfileController("facilitydata") - self.network_connection = self.controller.create_network_connection( - self.live_server_url - ) + self.network_connection = self.controller.create_network_connection(self.live_server_url) self.key = SharedKey.get_or_create_shared_key() @override_settings(MORANGO_INSTANCE_INFO={"this_is_a_test": "yes"}) @@ -123,15 +118,11 @@ def test_creating_sync_session_successful(self, mock_object): def test_creating_sync_session_cert_fails_to_verify(self, mock_verify, mock_create): mock_create.return_value.json.return_value = {} with self.assertRaises(CertificateSignatureInvalid): - self.network_connection.create_sync_session( - self.subset_cert, self.root_cert - ) + self.network_connection.create_sync_session(self.subset_cert, self.root_cert) def test_get_remote_certs(self): certs = self.subset_cert.get_ancestors(include_self=True) - remote_certs = self.network_connection.get_remote_certificates( - self.root_cert.id - ) + remote_certs = self.network_connection.get_remote_certificates(self.root_cert.id) self.assertSetEqual(set(certs), set(remote_certs)) @mock.patch.object(SessionWrapper, "request") @@ -148,9 +139,7 @@ def test_csr(self, mock_request): return_value=self.subset_cert.private_key.get_private_key_string(), ): self.network_connection.certificate_signing_request(self.root_cert, "", "") - self.assertTrue( - Certificate.objects.filter(id=json.loads(cert_serialized)["id"]).exists() - ) + self.assertTrue(Certificate.objects.filter(id=json.loads(cert_serialized)["id"]).exists()) @override_settings(ALLOW_CERTIFICATE_PUSHING=True) def test_push_signed_client_certificate_chain(self): @@ -228,9 +217,7 @@ def create(**data): mock_create.side_effect = create self.assertEqual(SyncSession.objects.filter(active=True).count(), 0) - client = self.network_connection.create_sync_session( - self.subset_cert, self.root_cert - ) + client = self.network_connection.create_sync_session(self.subset_cert, self.root_cert) self.assertEqual(SyncSession.objects.filter(active=True).count(), 1) self.network_connection.close_sync_session(client.sync_session) @@ -323,9 +310,7 @@ def test_initiate_pull(self, MockPullClient): filter = Filter("abc123") self.client.initiate_pull(filter) - MockPullClient.assert_called_with( - self.conn, self.session, self.client.controller - ) + MockPullClient.assert_called_with(self.conn, self.session, self.client.controller) mock_pull_client.initialize.assert_called_once_with(filter) mock_pull_client.run.assert_called_once() @@ -343,9 +328,7 @@ def test_initiate_push(self, MockPushClient): sync_filter = Filter("abc123") self.client.initiate_push(sync_filter) - MockPushClient.assert_called_with( - self.conn, self.session, self.client.controller - ) + MockPushClient.assert_called_with(self.conn, self.session, self.client.controller) mock_pull_client.initialize.assert_called_once_with(sync_filter) mock_pull_client.run.assert_called_once() @@ -365,8 +348,12 @@ def test_close_sync_session(self): class TransferClientTestCase(BaseTransferClientTestCase): def build_client(self, client_class=TransferClient, controller=None, update_context=False): - self.controller = controller or mock.Mock(spec="morango.sync.controller.SessionController")() - return super(TransferClientTestCase, self).build_client(client_class=client_class, controller=self.controller, update_context=update_context) + self.controller = ( + controller or mock.Mock(spec="morango.sync.controller.SessionController")() + ) + return super(TransferClientTestCase, self).build_client( + client_class=client_class, controller=self.controller, update_context=update_context + ) def test_init(self): self.assertIsInstance(self.client, TransferClient) @@ -407,9 +394,7 @@ def test_initialize(self, mock_proceed): sync_filter = self.transfer_session.get_filter() self.client.initialize(sync_filter) self.assertEqual(sync_filter, self.client.context.filter) - mock_proceed.assert_any_call( - transfer_stages.INITIALIZING, error_msg=mock.ANY - ) + mock_proceed.assert_any_call(transfer_stages.INITIALIZING, error_msg=mock.ANY) self.client.context.transfer_session = None self.client.context.children[0].transfer_session = self.transfer_session self.client.context.join(self.client.context.children[0]) @@ -439,7 +424,8 @@ def test_run(self): mock_end.assert_called_once() self.controller.proceed_to_and_wait_for.assert_any_call( - transfer_stages.TRANSFERRING, callback=mock.ANY, + transfer_stages.TRANSFERRING, + callback=mock.ANY, ) mock_fire = self.controller.proceed_to_and_wait_for.call_args_list[0][1].get("callback") mock_fire() diff --git a/tests/testapp/tests/sync/test_utils.py b/tests/testapp/tests/sync/test_utils.py index 04dca74a..6077ee77 100644 --- a/tests/testapp/tests/sync/test_utils.py +++ b/tests/testapp/tests/sync/test_utils.py @@ -1,8 +1,7 @@ import mock from django.test import TestCase -from morango.sync.utils import SyncSignal -from morango.sync.utils import SyncSignalGroup +from morango.sync.utils import SyncSignal, SyncSignalGroup class SyncSignalTestCase(TestCase): @@ -70,9 +69,7 @@ def test_send(self): with signaler.send(other="A") as status: start_handler.assert_called_once_with(this_is_a_default=True, other="A") status.in_progress.fire(this_is_a_default=False, other="B") - in_progress_handler.assert_called_once_with( - this_is_a_default=False, other="B" - ) + in_progress_handler.assert_called_once_with(this_is_a_default=False, other="B") completed_handler.assert_not_called() completed_handler.assert_called_once_with(this_is_a_default=True, other="A") diff --git a/tests/testapp/tests/test_api.py b/tests/testapp/tests/test_api.py index 2b1367d5..b2c8a306 100644 --- a/tests/testapp/tests/test_api.py +++ b/tests/testapp/tests/test_api.py @@ -3,8 +3,7 @@ from base64 import encodebytes as b64encode from django.db import connection -from django.test.utils import CaptureQueriesContext -from django.test.utils import override_settings +from django.test.utils import CaptureQueriesContext, override_settings from django.urls import reverse from django.urls.exceptions import NoReverseMatch from django.utils import timezone @@ -12,27 +11,24 @@ from facility_profile.models import MyUser from rest_framework.test import APITestCase -from .compat import EnvironmentVarGuard -from morango.api.serializers import BufferSerializer -from morango.api.serializers import CertificateSerializer -from morango.api.serializers import InstanceIDSerializer -from morango.constants import transfer_stages -from morango.constants import transfer_statuses -from morango.models.certificates import Certificate -from morango.models.certificates import Key -from morango.models.certificates import Nonce -from morango.models.certificates import ScopeDefinition -from morango.models.core import Buffer -from morango.models.core import DatabaseMaxCounter -from morango.models.core import InstanceIDModel -from morango.models.core import RecordMaxCounterBuffer -from morango.models.core import SyncSession -from morango.models.core import TransferSession +from morango.api.serializers import BufferSerializer, CertificateSerializer, InstanceIDSerializer +from morango.constants import transfer_stages, transfer_statuses +from morango.models.certificates import Certificate, Key, Nonce, ScopeDefinition +from morango.models.core import ( + Buffer, + DatabaseMaxCounter, + InstanceIDModel, + RecordMaxCounterBuffer, + SyncSession, + TransferSession, +) from morango.models.fields.crypto import SharedKey from morango.registry import syncable_models from morango.sync.syncsession import compress_string from morango.sync.utils import validate_and_create_buffer_data +from .compat import EnvironmentVarGuard + class CertificateTestCaseMixin(object): def setUp(self): @@ -74,9 +70,7 @@ def setUp(self): read_write_filter_template="", ) - self.root_cert1_with_key = Certificate.generate_root_certificate( - self.root_scope_def.id - ) + self.root_cert1_with_key = Certificate.generate_root_certificate(self.root_scope_def.id) self.subset_cert1_without_key = Certificate( parent=self.root_cert1_with_key, @@ -108,9 +102,7 @@ def setUp(self): self.subset_cert1_without_key._private_key = None self.subset_cert1_without_key.save() - self.root_cert2_without_key = Certificate.generate_root_certificate( - self.root_scope_def.id - ) + self.root_cert2_without_key = Certificate.generate_root_certificate(self.root_scope_def.id) self.subset_cert2_with_key = Certificate( parent=self.root_cert2_without_key, @@ -196,9 +188,7 @@ def create_syncsession(self, client_certificate=None, server_certificate=None): ), "connection_path": "http://127.0.0.1:8000", "instance": json.dumps( - InstanceIDSerializer( - InstanceIDModel.get_or_create_current_instance()[0] - ).data + InstanceIDSerializer(InstanceIDModel.get_or_create_current_instance()[0]).data ), "nonce": nonce, } @@ -213,13 +203,7 @@ def create_syncsession(self, client_certificate=None, server_certificate=None): return SyncSession.objects.get(id=data["id"]) def make_transfersession_creation_request( - self, - filter, - push, - syncsession=None, - expected_status=201, - expected_message=None, - **kwargs + self, filter, push, syncsession=None, expected_status=201, expected_message=None, **kwargs ): if not syncsession: @@ -234,9 +218,7 @@ def make_transfersession_creation_request( } # make the API call to attempt to create the TransferSession - response = self.client.post( - reverse("transfersessions-list"), data, format="json" - ) + response = self.client.post(reverse("transfersessions-list"), data, format="json") self.assertEqual(response.status_code, expected_status) if expected_status == 201: @@ -274,9 +256,7 @@ def test_certificate_filtering_by_primary_partition(self): self.assertEqual(data[0]["id"], self.subset_cert2_with_key.id) # check that no certificates are returned when the partition doesn't exist - _, data = self.make_cert_endpoint_request( - params={"primary_partition": "a" * 32} - ) + _, data = self.make_cert_endpoint_request(params={"primary_partition": "a" * 32}) self.assertEqual(len(data), 0) # check that no certificates are returned when profile doesn't match @@ -360,9 +340,7 @@ def make_csr(self, parent, **kwargs): params = { "parent": parent.id, "profile": kwargs.get("profile", self.profile), - "scope_definition": kwargs.get( - "scope_definition", parent.scope_definition_id - ), + "scope_definition": kwargs.get("scope_definition", parent.scope_definition_id), "scope_version": kwargs.get("scope_version", parent.scope_version), "scope_params": kwargs.get("scope_params", parent.scope_params), "public_key": kwargs.get("public_key", key.get_public_key_string()), @@ -436,9 +414,7 @@ def test_certificate_chain_pushing(self): [self.unsaved_root_cert, self.unsaved_subset_cert], many=True ).data ) - response = self.client.post( - reverse("certificatechain-list"), data=data, format="json" - ) + response = self.client.post(reverse("certificatechain-list"), data=data, format="json") self.assertEqual(response.status_code, 201) saved_subset_cert = Certificate.objects.get(id=self.unsaved_subset_cert.id) self.assertEqual( @@ -459,9 +435,7 @@ def test_certificate_chain_pushing_fails_for_nonce(self): [self.unsaved_root_cert, self.unsaved_subset_cert], many=True ).data ) - response = self.client.post( - reverse("certificatechain-list"), data=data, format="json" - ) + response = self.client.post(reverse("certificatechain-list"), data=data, format="json") self.assertEqual(response.status_code, 403) @override_settings(ALLOW_CERTIFICATE_PUSHING=True) @@ -477,9 +451,7 @@ def test_certificate_chain_pushing_fails_for_public_key(self): [self.unsaved_root_cert, self.unsaved_subset_cert], many=True ).data ) - response = self.client.post( - reverse("certificatechain-list"), data=data, format="json" - ) + response = self.client.post(reverse("certificatechain-list"), data=data, format="json") self.assertEqual(response.status_code, 400) @@ -532,17 +504,13 @@ def get_initial_syncsession_data_for_request(self): ), "connection_path": "http://127.0.0.1:8000", "instance": json.dumps( - InstanceIDSerializer( - InstanceIDModel.get_or_create_current_instance()[0] - ).data + InstanceIDSerializer(InstanceIDModel.get_or_create_current_instance()[0]).data ), "nonce": nonce, } # sign the nonce/ID combo to attach to the request - data["signature"] = self.sub_subset_cert1_with_key.sign( - "{nonce}:{id}".format(**data) - ) + data["signature"] = self.sub_subset_cert1_with_key.sign("{nonce}:{id}".format(**data)) return data @@ -578,12 +546,8 @@ def test_syncsession_can_be_created(self): # check that the syncsession was created syncsession = SyncSession.objects.get() self.assertEqual(syncsession.id, data["id"]) - self.assertEqual( - syncsession.server_certificate_id, data["server_certificate_id"] - ) - self.assertEqual( - syncsession.client_certificate_id, data["client_certificate_id"] - ) + self.assertEqual(syncsession.server_certificate_id, data["server_certificate_id"]) + self.assertEqual(syncsession.client_certificate_id, data["client_certificate_id"]) self.assertTrue(syncsession.active) def test_syncsession_creation_fails_with_bad_signature(self): @@ -611,9 +575,7 @@ def test_syncsession_creation_fails_with_expired_nonce(self): data = self.get_initial_syncsession_data_for_request() Nonce.objects.all().update( - timestamp=timezone.datetime( - 2000, 1, 1, tzinfo=timezone.get_current_timezone() - ) + timestamp=timezone.datetime(2000, 1, 1, tzinfo=timezone.get_current_timezone()) ) self.assertSyncSessionCreationFails(data) @@ -667,9 +629,7 @@ def test_get_sync_session(self): data = self.get_initial_syncsession_data_for_request() self.client.post(reverse("syncsessions-list"), data, format="json") - response = self.client.get( - reverse("syncsessions-detail", kwargs={"pk": data["id"]}) - ) + response = self.client.get(reverse("syncsessions-detail", kwargs={"pk": data["id"]})) self.assertEqual(response.status_code, 200) @@ -684,9 +644,7 @@ def test_transfersession_can_be_created(self): def test_transfersession_can_be_created_with_smaller_subset_filter(self): self.make_transfersession_creation_request( - filter=str(self.sub_subset_cert1_with_key.get_scope().read_filter).split()[ - 0 - ], + filter=str(self.sub_subset_cert1_with_key.get_scope().read_filter).split()[0], push=False, ) @@ -833,12 +791,8 @@ def test_inactive_transfersession_cannot_be_deleted(self): class BufferEndpointTestCase(CertificateTestCaseMixin, APITestCase): def setUp(self): super(BufferEndpointTestCase, self).setUp() - self.default_push_filter = str( - self.sub_subset_cert1_with_key.get_scope().write_filter - ) - self.default_pull_filter = str( - self.sub_subset_cert1_with_key.get_scope().read_filter - ) + self.default_push_filter = str(self.sub_subset_cert1_with_key.get_scope().write_filter) + self.default_pull_filter = str(self.sub_subset_cert1_with_key.get_scope().read_filter) def build_buffer_item(self, **kwargs): @@ -854,17 +808,11 @@ def build_buffer_item(self, **kwargs): kwargs["transfer_session"].records_total = records_total + 1 kwargs["transfer_session"].save() - filt = ( - server_cert.get_scope().write_filter - if push - else server_cert.get_scope().read_filter - ) + filt = server_cert.get_scope().write_filter if push else server_cert.get_scope().read_filter partition = filt._filter_tuple[0] + ":furthersubpart" data = { - "profile": kwargs.get( - "profile", kwargs["transfer_session"].sync_session.profile - ), + "profile": kwargs.get("profile", kwargs["transfer_session"].sync_session.profile), "serialized": kwargs.get("serialized", '{"test": 99}'), "deleted": kwargs.get("deleted", False), "last_saved_instance": kwargs.get("last_saved_instance", uuid.uuid4().hex), @@ -872,9 +820,7 @@ def build_buffer_item(self, **kwargs): "partition": kwargs.get("partition", partition), "source_id": kwargs.get("source_id", uuid.uuid4().hex), "model_name": kwargs.get("model_name", "contentsummarylog"), - "conflicting_serialized_data": kwargs.get( - "conflicting_serialized_data", "" - ), + "conflicting_serialized_data": kwargs.get("conflicting_serialized_data", ""), "model_uuid": kwargs.get("model_uuid", None), "transfer_session": kwargs["transfer_session"], } @@ -927,12 +873,10 @@ def make_buffer_post_request(self, buffers, expected_status=201, gzip=False): def test_push_valid_gzipped_buffer_chunk(self): rec_1 = self.build_buffer_item(push=True, filter=self.default_push_filter) rec_2 = self.build_buffer_item( - serialized=u"unicode", transfer_session=rec_1.transfer_session + serialized="unicode", transfer_session=rec_1.transfer_session ) rec_3 = self.build_buffer_item(transfer_session=rec_1.transfer_session) - self.make_buffer_post_request( - [rec_1, rec_2, rec_3], expected_status=201, gzip=True - ) + self.make_buffer_post_request([rec_1, rec_2, rec_3], expected_status=201, gzip=True) def test_push_valid_buffer_chunk(self): rec_1 = self.build_buffer_item(push=True, filter=self.default_push_filter) @@ -982,21 +926,15 @@ def create_records_for_pulling(self, count=3, **kwargs): records.append(self.build_buffer_item(transfer_session=transfer_session)) # also make some dummy records so we can make sure they don't get returned - records.append( - self.build_buffer_item(push=False, filter=self.default_pull_filter) - ) - records.append( - self.build_buffer_item(transfer_session=records[-1].transfer_session) - ) + records.append(self.build_buffer_item(push=False, filter=self.default_pull_filter)) + records.append(self.build_buffer_item(transfer_session=records[-1].transfer_session)) # save all the records to the database [rec.save() for rec in records] return records[0].transfer_session.id - def make_buffer_get_request( - self, expected_status=200, expected_count=None, **get_params - ): + def make_buffer_get_request(self, expected_status=200, expected_count=None, **get_params): """Make a GET request to the buffer endpoint. Warning: Deletes the local buffer instances before validating.""" response = self.client.get(reverse("buffers-list"), get_params, format="json") @@ -1004,7 +942,6 @@ def make_buffer_get_request( self.assertEqual(response.status_code, expected_status) if expected_status == 200: - t_id = get_params.get("transfer_session_id") if expected_count is None: @@ -1021,22 +958,16 @@ def make_buffer_get_request( model_uuids = [d["model_uuid"] for d in data] # delete "local" buffer records to avoid uniqueness constraint failures in validation - Buffer.objects.filter( - transfer_session_id=t_id, model_uuid__in=model_uuids - ).delete() + Buffer.objects.filter(transfer_session_id=t_id, model_uuid__in=model_uuids).delete() # run the validation logic to ensure no errors were returned - errors = validate_and_create_buffer_data( - data, TransferSession.objects.get(id=t_id) - ) + errors = validate_and_create_buffer_data(data, TransferSession.objects.get(id=t_id)) self.assertFalse(errors) # check that the correct number of buffer items were created self.assertEqual( expected_count, - Buffer.objects.filter( - transfer_session_id=t_id, model_uuid__in=model_uuids - ).count(), + Buffer.objects.filter(transfer_session_id=t_id, model_uuid__in=model_uuids).count(), ) # check that the correct number of buffer items was returned @@ -1053,7 +984,7 @@ def test_buffer_serializer_makes_no_transfer_session_query(self): with CaptureQueriesContext(connection) as ctx: BufferSerializer(instance=buffers[0]).data for q in ctx.captured_queries: - self.assertFalse('morango_transfersession' in q['sql']) + self.assertFalse("morango_transfersession" in q["sql"]) def test_pull_valid_buffer_list(self): @@ -1137,18 +1068,14 @@ def test_pull_by_page_offset__order_by(self): limit=5, offset=offset, ) - response = self.client.get( - reverse("buffers-list"), get_params, format="json" - ) + response = self.client.get(reverse("buffers-list"), get_params, format="json") self.assertEqual(response.status_code, 200) data = json.loads(response.content.decode()) model_uuids = {d["model_uuid"] for d in data["results"]} self.assertFalse(model_uuids & returned_uuids) returned_uuids.update(model_uuids) if last_transfer_session_id: - Buffer.objects.filter( - transfer_session_id=last_transfer_session_id - ).delete() + Buffer.objects.filter(transfer_session_id=last_transfer_session_id).delete() last_transfer_session_id = self.create_records_for_pulling(count=10) offset += 5 @@ -1172,9 +1099,7 @@ def test_id_changes_id_hash_changes(self): with EnvironmentVarGuard() as env: env["MORANGO_SYSTEM_ID"] = "new_sys_id" InstanceIDModel.get_or_create_current_instance(clear_cache=True) - m_info = self.client.get( - reverse("morangoinfo-detail", kwargs={"pk": 1}), format="json" - ) + m_info = self.client.get(reverse("morangoinfo-detail", kwargs={"pk": 1}), format="json") self.assertNotEqual(m_info.data["instance_hash"], old_id_hash) @override_settings(MORANGO_INSTANCE_INFO={"this_is_a_test": "yes"}) @@ -1185,9 +1110,7 @@ def test_custom_instance_info(self): self.assertIn("this_is_a_test", self.m_info.data) self.assertEqual(self.m_info.data["this_is_a_test"], "yes") - @override_settings( - MORANGO_INSTANCE_INFO="facility_profile.custom:CUSTOM_INSTANCE_INFO" - ) + @override_settings(MORANGO_INSTANCE_INFO="facility_profile.custom:CUSTOM_INSTANCE_INFO") def test_custom_instance_info__import_path(self): self.m_info = self.client.get( reverse("morangoinfo-detail", kwargs={"pk": 1}), format="json" diff --git a/tests/testapp/tests/test_management_commands.py b/tests/testapp/tests/test_management_commands.py index 9428c94e..f93be95b 100644 --- a/tests/testapp/tests/test_management_commands.py +++ b/tests/testapp/tests/test_management_commands.py @@ -5,16 +5,14 @@ from django.test import TestCase from django.utils import timezone +from morango.models.core import SyncSession, TransferSession + from .helpers import create_buffer_and_store_dummy_data -from morango.models.core import SyncSession -from morango.models.core import TransferSession def _create_sessions(last_activity_offset=0, sync_session=None, push=True): - last_activity_timestamp = timezone.now() - datetime.timedelta( - hours=last_activity_offset - ) + last_activity_timestamp = timezone.now() - datetime.timedelta(hours=last_activity_offset) if sync_session is None: sync_session = SyncSession.objects.create( @@ -30,7 +28,7 @@ def _create_sessions(last_activity_offset=0, sync_session=None, push=True): sync_session=sync_session, push=push, last_activity_timestamp=last_activity_timestamp, - filter="1:2\n" + filter="1:2\n", ) return sync_session, transfer_session @@ -104,14 +102,18 @@ def test_filtering_sessions_cleared(self): self.assertSyncSessionIsActive(self.syncsession_new) def test_filtering_sessions_by_client_instance_id_cleared(self): - call_command("cleanupsyncs", client_instance_id=self.syncsession_old.client_instance_id, expiration=0) + call_command( + "cleanupsyncs", client_instance_id=self.syncsession_old.client_instance_id, expiration=0 + ) self.assertTransferSessionIsCleared(self.transfersession_old) self.assertSyncSessionIsNotActive(self.syncsession_old) self.assertTransferSessionIsNotCleared(self.transfersession_new) self.assertSyncSessionIsActive(self.syncsession_new) def test_filtering_sessions_by_server_instance_id_cleared(self): - call_command("cleanupsyncs", server_instance_id=self.syncsession_old.server_instance_id, expiration=0) + call_command( + "cleanupsyncs", server_instance_id=self.syncsession_old.server_instance_id, expiration=0 + ) self.assertTransferSessionIsCleared(self.transfersession_old) self.assertSyncSessionIsNotActive(self.syncsession_old) self.assertTransferSessionIsNotCleared(self.transfersession_new) diff --git a/tests/testapp/tests/test_migrations.py b/tests/testapp/tests/test_migrations.py index 6b8068d1..fdf80cc8 100644 --- a/tests/testapp/tests/test_migrations.py +++ b/tests/testapp/tests/test_migrations.py @@ -4,8 +4,7 @@ from django.conf import settings from django.db import connection from django.db.utils import IntegrityError -from django.test import TestCase -from django.test import TransactionTestCase +from django.test import TestCase, TransactionTestCase from django.utils import timezone from .helpers import TestMigrationsMixin @@ -27,8 +26,12 @@ def setUpBeforeMigration(self, apps): SyncSession = apps.get_model("morango", "SyncSession") with connection.cursor() as cursor: - cursor.execute("ALTER TABLE morango_transfersession ALTER COLUMN transfer_stage SET NOT NULL") - cursor.execute("ALTER TABLE morango_transfersession ALTER COLUMN transfer_stage_status SET NOT NULL") + cursor.execute( + "ALTER TABLE morango_transfersession ALTER COLUMN transfer_stage SET NOT NULL" + ) + cursor.execute( + "ALTER TABLE morango_transfersession ALTER COLUMN transfer_stage_status SET NOT NULL" + ) self.sync_session = SyncSession.objects.create( id=uuid.uuid4().hex, @@ -68,7 +71,9 @@ class SkipIfExistsMigrationTest(TestMigrationsMixin, TransactionTestCase): def setUpBeforeMigration(self, apps): # simulate as if we already created an index on the partition field of the Store model with connection.cursor() as cursor: - cursor.execute('CREATE INDEX "idx_morango_store_partition" ON "morango_store" ("partition" text_pattern_ops);') + cursor.execute( + 'CREATE INDEX "idx_morango_store_partition" ON "morango_store" ("partition" text_pattern_ops);' + ) def test_runs(self): """ diff --git a/tests/testapp/tests/test_utils.py b/tests/testapp/tests/test_utils.py index 3e9ad430..782c5b64 100644 --- a/tests/testapp/tests/test_utils.py +++ b/tests/testapp/tests/test_utils.py @@ -4,27 +4,29 @@ import pytest from django.http.request import HttpRequest from django.test.testcases import SimpleTestCase -from facility_profile.models import Facility -from facility_profile.models import MyUser +from facility_profile.models import Facility, MyUser from requests import Request from morango.constants import transfer_stages -from morango.constants.capabilities import ALLOW_CERTIFICATE_PUSHING -from morango.constants.capabilities import ASYNC_OPERATIONS -from morango.constants.capabilities import FSIC_V2_FORMAT -from morango.utils import _posix_pid_exists -from morango.utils import _windows_pid_exists -from morango.utils import CAPABILITIES_CLIENT_HEADER -from morango.utils import get_capabilities -from morango.utils import parse_capabilities_from_server_request -from morango.utils import pid_exists -from morango.utils import self_referential_fk -from morango.utils import serialize_capabilities_to_client_request -from morango.utils import SETTINGS +from morango.constants.capabilities import ( + ALLOW_CERTIFICATE_PUSHING, + ASYNC_OPERATIONS, + FSIC_V2_FORMAT, +) +from morango.utils import ( + CAPABILITIES_CLIENT_HEADER, + SETTINGS, + _posix_pid_exists, + _windows_pid_exists, + get_capabilities, + parse_capabilities_from_server_request, + pid_exists, + self_referential_fk, + serialize_capabilities_to_client_request, +) class SettingsTestCase(SimpleTestCase): - def assertLength(self, expected, iterable): self.assertEqual(expected, len(iterable)) diff --git a/tests/testapp/tests/test_uuid_utilities.py b/tests/testapp/tests/test_uuid_utilities.py index de48dfd1..892b6764 100644 --- a/tests/testapp/tests/test_uuid_utilities.py +++ b/tests/testapp/tests/test_uuid_utilities.py @@ -3,19 +3,19 @@ import mock from django.test import TestCase -from facility_profile.models import Facility -from facility_profile.models import InteractionLog -from facility_profile.models import MyUser +from facility_profile.models import Facility, InteractionLog, MyUser -from .compat import EnvironmentVarGuard from morango.errors import InvalidMorangoSourceId -from morango.models.core import DatabaseIDModel -from morango.models.core import InstanceIDModel +from morango.models.core import DatabaseIDModel, InstanceIDModel from morango.models.fields.uuids import sha2_uuid -from morango.models.utils import _calculate_0_4_uuid -from morango.models.utils import get_0_4_system_parameters -from morango.models.utils import get_0_5_mac_address -from morango.models.utils import get_0_5_system_id +from morango.models.utils import ( + _calculate_0_4_uuid, + get_0_4_system_parameters, + get_0_5_mac_address, + get_0_5_system_id, +) + +from .compat import EnvironmentVarGuard class UUIDModelMixinTestCase(TestCase): @@ -24,9 +24,7 @@ def setUp(self): def test_calculate_uuid(self): log_with_random_id = InteractionLog(user=MyUser.objects.create(username="Test")) - with mock.patch( - "uuid.uuid4", return_value=uuid.UUID("12345678123456781234567812345678") - ): + with mock.patch("uuid.uuid4", return_value=uuid.UUID("12345678123456781234567812345678")): target_uuid = sha2_uuid( log_with_random_id.calculate_partition(), "12345678123456781234567812345678", @@ -74,32 +72,24 @@ def test_only_one_current_instance_ID(self): self.assertEqual(len(InstanceIDModel.objects.filter(current=True)), 1) def test_same_node_id(self): - with mock.patch( - "uuid.getnode", return_value=67002173923623 - ): # fake (random) address + with mock.patch("uuid.getnode", return_value=67002173923623): # fake (random) address (IDModel, _) = InstanceIDModel.get_or_create_current_instance() ident = IDModel.id - with mock.patch( - "uuid.getnode", return_value=69002173923623 - ): # fake (random) address + with mock.patch("uuid.getnode", return_value=69002173923623): # fake (random) address (IDModel, _) = InstanceIDModel.get_or_create_current_instance() - with mock.patch( - "uuid.getnode", return_value=67002173923623 - ): # fake (random) address + with mock.patch("uuid.getnode", return_value=67002173923623): # fake (random) address (IDModel, _) = InstanceIDModel.get_or_create_current_instance() - self.assertFalse( - InstanceIDModel.objects.exclude(id=ident).filter(current=True).exists() - ) + self.assertFalse(InstanceIDModel.objects.exclude(id=ident).filter(current=True).exists()) self.assertTrue(InstanceIDModel.objects.get(id=ident).current) @mock.patch("uuid.getnode", return_value=24359248572014) @mock.patch("platform.platform", return_value="Windows 3.1") @mock.patch("platform.node", return_value="myhost") @mock.patch("morango.models.utils._get_database_path", return_value="") - @mock.patch('sys.version', '2.7.333') + @mock.patch("sys.version", "2.7.333") def test_consistent_with_0_4_instance_id_calculation(self, *args): """ This test ensures that we don't accidentally make changes that impact how we calculate @@ -155,9 +145,7 @@ def test_consistent_0_5_instance_id(self, *args): env["MORANGO_SYSTEM_ID"] = "magicsysid" DatabaseIDModel.objects.all().update(current=False) - DatabaseIDModel.objects.create( - id="7fe445b75cea11858c00fb97bdee8878", current=True - ) + DatabaseIDModel.objects.create(id="7fe445b75cea11858c00fb97bdee8878", current=True) self.assertEqual(get_0_5_system_id(), "54940f560a55bbf7d86b") self.assertEqual(get_0_5_mac_address(), "804f4c20d3b2b5a29b95") @@ -205,7 +193,6 @@ def test_instance_id_caching(self, *args): """ with EnvironmentVarGuard() as env: - env["MORANGO_SYSTEM_ID"] = "oldmagicsysid" old_instance, created = InstanceIDModel.get_or_create_current_instance(clear_cache=True) @@ -216,7 +203,9 @@ def test_instance_id_caching(self, *args): cached_instance, created = InstanceIDModel.get_or_create_current_instance() self.assertFalse(created) - uncached_instance, created = InstanceIDModel.get_or_create_current_instance(clear_cache=True) + uncached_instance, created = InstanceIDModel.get_or_create_current_instance( + clear_cache=True + ) self.assertTrue(created) recached_instance, created = InstanceIDModel.get_or_create_current_instance() diff --git a/uv.lock b/uv.lock index 5bd5f96a..b6445220 100644 --- a/uv.lock +++ b/uv.lock @@ -12,6 +12,10 @@ resolution-markers = [ "python_full_version < '3.6.8'", ] +[options] +exclude-newer = "2026-03-26T07:53:50.714888877Z" +exclude-newer-span = "P7D" + [[package]] name = "alabaster" version = "0.7.13"