import logging import time import traceback from typing import Dict, List, Tuple, Optional from django.db import transaction from django.core.exceptions import ValidationError from django.utils import timezone from organization.models import Organization, SyncLog, SyncConfiguration from organization.services.external_api_service import ExternalAPIService, APIError logger = logging.getLogger(__name__) class DataMappingError(Exception): """Raised when data mapping fails.""" pass class SyncError(Exception): """Raised when sync operation fails.""" pass class OrganizationSyncService: """ Service for syncing organization data from external API. Handles data mapping, upsert logic, and related data creation. """ def __init__(self, config: SyncConfiguration): """ Initialize sync service. Args: config: SyncConfiguration instance """ self.config = config self.api_service = ExternalAPIService(config) self.logger = logger self.logger.info(f"Initialized OrganizationSyncService for {config.source_name}") def _map_external_data(self, external_record: Dict) -> Dict: """ Map external API response to Organization model fields. Args: external_record: Raw data from external API Returns: Mapped data ready for Organization model Raises: DataMappingError: If mapping fails """ try: # Use custom field mapping if configured field_mapping = self.config.field_mapping or {} mapped_data = { "external_id": str(external_record.get("id", "")), "external_source": self.config.source_name, "external_metadata": external_record, # Store raw data as metadata } # Map required fields if field_mapping: # Use custom mapping for org_field, ext_field in field_mapping.items(): if ext_field in external_record: mapped_data[org_field] = external_record[ext_field] else: # Use default field mapping default_mapping = { "name": "name", "code": "code", "email": "email", "phone": "phone", "website": "website", "description": "description", "address_line1": "address_line1", "address_line2": "address_line2", "city": "city", "state": "state", "postal_code": "postal_code", "country": "country", } for org_field, ext_field in default_mapping.items(): if ext_field in external_record: mapped_data[org_field] = external_record[ext_field] # Ensure required fields exist if not mapped_data.get("name"): raise DataMappingError("'name' field is required and not found in mapping") if not mapped_data.get("code"): raise DataMappingError("'code' field is required and not found in mapping") if not mapped_data.get("email"): raise DataMappingError("'email' field is required and not found in mapping") return mapped_data except DataMappingError: raise except Exception as e: raise DataMappingError(f"Failed to map external data: {str(e)}") def _validate_organization_data(self, data: Dict) -> Tuple[bool, Optional[str]]: """ Validate organization data before saving. Args: data: Organization data Returns: Tuple of (is_valid, error_message) """ errors = [] # Check required fields if not data.get("name"): errors.append("'name' is required") if not data.get("code"): errors.append("'code' is required") if not data.get("email"): errors.append("'email' is required") # Validate email format from django.core.validators import validate_email if data.get("email"): try: validate_email(data["email"]) except ValidationError: errors.append(f"Invalid email format: {data['email']}") # Validate URL if present if data.get("website"): from django.core.validators import URLValidator validator = URLValidator() try: validator(data["website"]) except ValidationError: errors.append(f"Invalid URL format: {data['website']}") return len(errors) == 0, "; ".join(errors) if errors else None def _create_or_update_organization(self, mapped_data: Dict) -> Tuple[Organization, bool]: """ Create or update organization (upsert). Args: mapped_data: Mapped organization data Returns: Tuple of (organization, created) Raises: SyncError: If upsert fails """ try: external_id = mapped_data["external_id"] external_source = mapped_data["external_source"] # Try to find existing organization organization, created = Organization.objects.update_or_create( external_id=external_id, external_source=external_source, defaults={ k: v for k, v in mapped_data.items() if k not in ["external_id", "external_source"] } ) action = "created" if created else "updated" self.logger.info( f"Organization {action}: {organization.name} " f"(external_id: {external_id})" ) return organization, created except Exception as e: raise SyncError(f"Failed to create/update organization: {str(e)}") def _create_default_departments(self, organization: Organization) -> int: """ Create default departments for organization if they don't exist. Args: organization: Organization instance Returns: Number of departments created """ try: from departments.models import Department from status.models import Status default_departments = [ { "name": "IT & Computer Science", "code": "CS", }, { "name": "Business Administration", "code": "BAS", }, { "name": "Engineering", "code": "ENG", }, { "name": "Liberal Arts", "code": "LA", }, ] created_count = 0 active_status = Status.objects.filter(type="department", slug="active").first() for dept_data in default_departments: # Check if department already exists for this org # Note: If your Department model has an org FK, uncomment below # if not Department.objects.filter( # name=dept_data["name"], # organization=organization # ).exists(): if not Department.objects.filter( department_code=dept_data["code"] ).exists(): dept = Department.objects.create( department_name=dept_data["name"], department_code=dept_data["code"], status=active_status ) created_count += 1 self.logger.info( f"Created default department: {dept.department_name}" ) return created_count except Exception as e: self.logger.warning( f"Failed to create default departments: {str(e)}" ) return 0 def sync_organizations( self, sync_log: SyncLog, max_records: Optional[int] = None ) -> None: """ Sync all organizations from external API. Args: sync_log: SyncLog instance to record operation max_records: Maximum organizations to sync Raises: SyncError: If sync fails """ start_time = time.time() failed_records = [] try: self.logger.info(f"Starting sync for {self.config.source_name}") sync_log.status = SyncLog.SyncStatus.IN_PROGRESS sync_log.save() # Fetch all organizations from API organizations_list = list( self.api_service.fetch_paginated( batch_size=self.config.batch_size, max_records=max_records ) ) sync_log.total_fetched = len(organizations_list) self.logger.info(f"Fetched {sync_log.total_fetched} organizations from API") # Process each organization in a transaction with transaction.atomic(): for index, external_record in enumerate(organizations_list, 1): try: self.logger.debug(f"Processing record {index}/{sync_log.total_fetched}") # Map data mapped_data = self._map_external_data(external_record) # Validate data is_valid, error_msg = self._validate_organization_data(mapped_data) if not is_valid: self.logger.warning( f"Validation failed for record {index}: {error_msg}" ) sync_log.error_count += 1 failed_records.append({ "index": index, "external_id": external_record.get("id"), "error": error_msg }) continue # Create/update organization organization, created = self._create_or_update_organization( mapped_data ) # Update the organization's last_synced timestamp organization.mark_synced() if created: sync_log.created_count += 1 # Create default departments for new organizations self._create_default_departments(organization) else: sync_log.updated_count += 1 except DataMappingError as e: self.logger.error(f"Data mapping error for record {index}: {str(e)}") sync_log.error_count += 1 failed_records.append({ "index": index, "external_id": external_record.get("id"), "error": str(e) }) except SyncError as e: self.logger.error(f"Sync error for record {index}: {str(e)}") sync_log.error_count += 1 failed_records.append({ "index": index, "external_id": external_record.get("id"), "error": str(e) }) except Exception as e: self.logger.error( f"Unexpected error for record {index}: {str(e)}" ) sync_log.error_count += 1 failed_records.append({ "index": index, "external_id": external_record.get("id"), "error": f"Unexpected error: {str(e)}" }) # Mark skipped records sync_log.skipped_count = ( sync_log.total_fetched - sync_log.created_count - sync_log.updated_count - sync_log.error_count ) # Store failed records if failed_records: sync_log.failed_records = failed_records # Determine final status if sync_log.error_count == 0: sync_log.mark_completed(duration=time.time() - start_time) self.logger.info("Sync completed successfully") else: if sync_log.created_count + sync_log.updated_count > 0: sync_log.mark_partial(duration=time.time() - start_time) self.logger.warning("Sync completed with errors (partial success)") else: sync_log.mark_failed( "All records failed to sync", duration=time.time() - start_time ) self.logger.error("Sync failed completely") except APIError as e: duration = time.time() - start_time error_msg = f"API Error: {str(e)}" error_trace = traceback.format_exc() self.logger.error(f"API error during sync: {error_msg}") sync_log.mark_failed(error_msg, error_trace, duration) raise SyncError(error_msg) except transaction.TransactionManagementError as e: duration = time.time() - start_time error_msg = f"Transaction Error: {str(e)}" error_trace = traceback.format_exc() self.logger.error(f"Transaction error during sync: {error_msg}") sync_log.mark_failed(error_msg, error_trace, duration) raise SyncError(error_msg) except Exception as e: duration = time.time() - start_time error_msg = f"Unexpected Error: {str(e)}" error_trace = traceback.format_exc() self.logger.error(f"Unexpected error during sync: {error_msg}") sync_log.mark_failed(error_msg, error_trace, duration) raise SyncError(error_msg) def sync_single_organization( self, external_id: str, sync_log: SyncLog ) -> Organization: """ Sync a single organization by external ID. Args: external_id: External organization ID sync_log: SyncLog instance to record operation Returns: Synced organization Raises: SyncError: If sync fails """ start_time = time.time() try: self.logger.info(f"Starting sync for organization {external_id}") # Fetch single organization external_record = self.api_service.fetch_organization_by_id(external_id) # Map and validate mapped_data = self._map_external_data(external_record) is_valid, error_msg = self._validate_organization_data(mapped_data) if not is_valid: raise DataMappingError(f"Validation failed: {error_msg}") # Create/update with transaction.atomic(): organization, created = self._create_or_update_organization(mapped_data) organization.mark_synced() if created: sync_log.created_count = 1 self._create_default_departments(organization) else: sync_log.updated_count = 1 sync_log.total_fetched = 1 sync_log.mark_completed(duration=time.time() - start_time) self.logger.info( f"Successfully synced organization: {organization.name}" ) return organization except (DataMappingError, SyncError) as e: duration = time.time() - start_time sync_log.mark_failed(str(e), duration=duration) raise SyncError(f"Failed to sync organization {external_id}: {str(e)}") except APIError as e: duration = time.time() - start_time sync_log.mark_failed(f"API Error: {str(e)}", duration=duration) raise SyncError(f"API error while syncing {external_id}: {str(e)}") except Exception as e: duration = time.time() - start_time error_trace = traceback.format_exc() sync_log.mark_failed(f"Error: {str(e)}", error_trace, duration) raise SyncError(f"Unexpected error syncing {external_id}: {str(e)}")