diff --git a/bin/replicate_now.sh b/bin/replicate_now.sh new file mode 100755 index 0000000000..d83982ea6c --- /dev/null +++ b/bin/replicate_now.sh @@ -0,0 +1,225 @@ +#!/bin/bash +set -e + +# Colors +GREEN='\033[0;32m' +BLUE='\033[0;34m' +YELLOW='\033[1;33m' +RED='\033[0;31m' +NC='\033[0m' # No Color + +echo -e "${BLUE}--- MDS NoW Application Replicator ---${NC}" +echo "This script replicates a generic NoW Application from Production to your local environment OR a dev/test OpenShift environment." +echo "Prerequisites: 'oc' (logged into prod) and 'docker' must be installed." +echo "" + +# Check dependencies +if ! command -v oc &> /dev/null; then + echo -e "${RED}Error: 'oc' command not found. Please install OpenShift CLI.${NC}" + exit 1 +fi +if ! command -v docker &> /dev/null; then + echo -e "${RED}Error: 'docker' command not found. Please install Docker.${NC}" + exit 1 +fi + +# --- Helper Function: Find Primary Crunchy DB Pod --- +find_primary_db_pod() { + local namespace=$1 + local selector="-n $namespace" + if [ -z "$namespace" ]; then + selector="" + fi + + # Try to find the Crunchy Data Primary/Master pod + local candidates=$(oc get pods $selector -l postgres-operator.crunchydata.com/role=master -o custom-columns=":metadata.name" --no-headers 2>/dev/null | grep -v "clone" || true) + local master_pod=$(echo "$candidates" | grep "ha-" | head -n 1 || true) + master_pod=${master_pod:-$(echo "$candidates" | head -n 1)} + + if [ -z "$master_pod" ]; then + # Fallback: legacy search + master_pod=$(oc get pods $selector -L postgres-operator.crunchydata.com/role --no-headers 2>/dev/null | grep "master" | grep -v "clone" | head -n 1 | awk '{print $1}') + fi + echo "$master_pod" +} + +# --- Helper Function: Find Backend Pod --- +find_backend_pod() { + local namespace=$1 + # User specified we should look for core-api-*** but not core-api-celery-*** + # We list pods, grep for core-api, exclude celery, and take the first one. + local candidate=$(oc get pods -n "$namespace" -o name 2>/dev/null | grep "core-api" | grep -v "celery" | head -n 1 | cut -d/ -f2) + + # Fallback to generic backend if core-api not found (just in case) + if [ -z "$candidate" ]; then + candidate=$(oc get pods -n "$namespace" -l app.kubernetes.io/name=backend -o custom-columns=":metadata.name" --no-headers 2>/dev/null | head -n 1) + fi + + echo "$candidate" +} + + + +# --- Helper Function: Check OpenShift Login --- +check_login() { + if ! oc whoami &> /dev/null; then + echo -e "${YELLOW}You are not logged into OpenShift.${NC}" + echo -e "Please generate a token from: ${BLUE}https://oauth-openshift.apps.silver.devops.gov.bc.ca/oauth/token/request${NC}" + echo -e "${YELLOW}...Paste Token Here...${NC}" + read -s OC_TOKEN + echo "" + + if [ -z "$OC_TOKEN" ]; then + echo -e "${RED}Error: Token is required.${NC}" + exit 1 + fi + + echo "Logging in..." + oc login --token="$OC_TOKEN" --server=https://api.silver.devops.gov.bc.ca:6443 > /dev/null + + if ! oc whoami &> /dev/null; then + echo -e "${RED}Error: Login failed.${NC}" + exit 1 + fi + echo -e "${GREEN}Successfully logged in as $(oc whoami)${NC}" + else + echo -e "Logged in as: ${GREEN}$(oc whoami)${NC}" + fi +} + +echo "" +check_login +echo "" + +# 1. Prompt for NoW Number +read -p "Enter NoW Number to be copied from Prod (e.g. 0400022-2025-01): " NOW_NUMBER +if [ -z "$NOW_NUMBER" ]; then + echo -e "${RED}Error: NoW Number is required.${NC}" + exit 1 +fi + +# 2. Prompt for Target Environment +echo "" +echo "Select Target Environment:" +echo "1) Local (default)" + +# Fetch available OpenShift Namespaces in 4c2ba9, EXCLUDING prod +# Assuming the user has access to list projects. If not, we might need a manual entry or just fail gracefully. +PROJECTS=$(oc get projects --no-headers 2>/dev/null | awk '{print $1}' | grep "^4c2ba9-" | grep -vE "prod$|tools$" || true) + +i=2 +declare -A PROJECT_MAP +if [ -n "$PROJECTS" ]; then + for proj in $PROJECTS; do + echo "$i) $proj" + PROJECT_MAP[$i]=$proj + i=$((i+1)) + done +fi + +read -p "Select Target [1]: " TARGET_SELECTION +TARGET_SELECTION=${TARGET_SELECTION:-1} + +TARGET_ENV="local" +TARGET_NAMESPACE="" + +if [ "$TARGET_SELECTION" -ne 1 ]; then + TARGET_NAMESPACE=${PROJECT_MAP[$TARGET_SELECTION]} + if [ -z "$TARGET_NAMESPACE" ]; then + echo -e "${RED}Error: Invalid selection.${NC}" + exit 1 + fi + TARGET_ENV="openshift" + echo -e "Targeting OpenShift Namespace: ${GREEN}${TARGET_NAMESPACE}${NC}" +else + echo -e "Targeting: ${GREEN}Local Docker Environment${NC}" +fi + +# 3. Prompt for Mine GUID +echo "" +echo "Enter the GUID of the Mine in the TARGET environment you want to attach this NoW to." +read -p "Mine GUID: " MINE_GUID +if [ -z "$MINE_GUID" ]; then + echo -e "${RED}Error: Mine GUID is required.${NC}" + exit 1 +fi + +# 4. Prompt for SOURCE Prod Pod +echo "" +echo "Select the SOURCE Production Database Pod (must be in 4c2ba9-prod)." +SOURCE_NAMESPACE="4c2ba9-prod" + +# Auto-detect SOURCE pod in PROD namespace +MASTER_POD=$(find_primary_db_pod "$SOURCE_NAMESPACE") +PROD_POD=${MASTER_POD:-postgresql-prod-0} + +echo "Detected Primary Pod Candidate in PROD: ${MASTER_POD:-(none)}" +echo "" +echo "Available Postgres Pods in PROD:" +oc get pods -n "$SOURCE_NAMESPACE" -L postgres-operator.crunchydata.com/role --no-headers 2>/dev/null | grep -E "postgres|crunchy" | grep "ha-" || echo " (none found)" +echo "" + +read -p "Source DB Pod Name [$PROD_POD]: " USER_POD +PROD_POD=${USER_POD:-$PROD_POD} +if [ -z "$PROD_POD" ]; then + echo -e "${RED}Error: Source DB Pod not found or specified.${NC}" + exit 1 +fi + + +echo "" +echo -e "${YELLOW}Step 1: Generating SQL query from LOCAL COMPATIBLE backend...${NC}" +# We always generate the query using the LOCAL backend code to ensure compatibility with the logic we are running. +# NOTE: If we are targeting a remote env, we are sending data compatible with the LOCAL codebase. +# Ideally, the remote env code matches. +QUERY=$(docker compose exec -T backend flask generate-now-query --now-number "$NOW_NUMBER" | grep "SELECT 'now_application_guid=") + +if [ -z "$QUERY" ]; then + echo -e "${RED}Error: Failed to generate SQL query.${NC}" + exit 1 +fi + +echo "$QUERY" > generated_query.sql +echo "Debug: SQL query saved to generated_query.sql" + +echo -e "${YELLOW}Step 2: Fetching data from Prod and ingesting into Target...${NC}" + +if [ "$TARGET_ENV" == "local" ]; then + # --- Execute for Local --- + (echo "SET search_path TO public;"; cat generated_query.sql) | \ + oc exec -n "$SOURCE_NAMESPACE" -i "$PROD_POD" -c database -- psql -d mds -At | \ + docker compose exec -T backend flask create-test-data --scenario now-application --mine-guid "$MINE_GUID" --non-interactive +else + # --- Execute for Remote OpenShift --- + echo "Locating backend pod in ${TARGET_NAMESPACE}..." + TARGET_POD_CANDIDATE=$(find_backend_pod "$TARGET_NAMESPACE") + + echo "Detected Backend Pod Candidate: ${TARGET_POD_CANDIDATE:-(none)}" + echo "" + echo "Available Pods in Target Namespace (${TARGET_NAMESPACE}):" + echo "Available Pods in Target Namespace (${TARGET_NAMESPACE}):" + # List all pods to help user verify + oc get pods -n "$TARGET_NAMESPACE" --no-headers 2>/dev/null | grep "core-api" | grep -v "celery" || echo " (none found or access denied)" + echo "" + + read -p "Target Backend Pod Name [${TARGET_POD_CANDIDATE}]: " USER_TARGET_POD + TARGET_POD=${USER_TARGET_POD:-$TARGET_POD_CANDIDATE} + + if [ -z "$TARGET_POD" ]; then + echo -e "${RED}Error: Target Backend Pod not specified.${NC}" + rm -f generated_query.sql + exit 1 + fi + echo "Using target backend pod: $TARGET_POD" + + echo "Ingesting data..." + (echo "SET search_path TO public;"; cat generated_query.sql) | \ + oc exec -n "$SOURCE_NAMESPACE" -i "$PROD_POD" -c database -- psql -d mds -At | \ + oc exec -n "$TARGET_NAMESPACE" -i "$TARGET_POD" -- flask create-test-data --scenario now-application --mine-guid "$MINE_GUID" --non-interactive +fi + +rm -f generated_query.sql + +echo "" +echo -e "${GREEN}Done!${NC}" +echo -e "${GREEN}Done!${NC}" diff --git a/package.json b/package.json index af9a9d120c..f7f6b23be1 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,10 @@ "@syncfusion/ej2-pdfviewer": "31.2.5" }, "scripts": { - "postinstall": "husky install" + "postinstall": "husky install", + "core-web": "yarn workspace @mds/core-web serve", + "minespace-web": "yarn workspace @mds/minespace-web serve", + "frontend": "yarn core-web && yarn minespace-web" }, "lint-staged": { "*.{js,css,json,md,ts,tsx}": [ diff --git a/services/core-api/app/commands.py b/services/core-api/app/commands.py index 89d0d8cb9f..08d1747d18 100644 --- a/services/core-api/app/commands.py +++ b/services/core-api/app/commands.py @@ -1,3 +1,6 @@ +import re +import os +import uuid from multiprocessing.dummy import Pool as ThreadPool from typing import List @@ -11,6 +14,7 @@ from app.api.utils.include.user_info import User from app.config import Config from app.extensions import db +from app.api.constants import PERMIT_LINKED_CONTACT_TYPES from flask import current_app from sqlalchemy.exc import DBAPIError from tests.factories import ( @@ -19,8 +23,12 @@ MinespaceSubscriptionFactory, MinespaceUserFactory, NOWApplicationIdentityFactory, + NOWApplicationProgressFactory, + NOWApplicationReviewFactory, ) +DUMMY_SIGNATURE = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==" + from .cli_commands.generate_history_table_migration import ( generate_history_table_migration, generate_table_migration, @@ -413,6 +421,715 @@ def bulk_export_permit_conditions(csv_path, output_dir): with current_app.app_context(): bulk_export_permit_conditions(csv_path, output_dir) + CHUNKED_FIELDS = ('contacts_data', 'state_of_land_data', 'blasting_operation_data', 'now_application_progress_data', 'now_application_review_data', 'documents_data', 'permit_amendments_data') + + def process_override_input(paste_data): + # Unescape common characters + paste_data = paste_data.replace('\\n', '\n').replace('\\r', '\r') + + reassembled = {} + + # 1. Handle chunked fields: {key}=SEGMENT_START {content} SEGMENT_END + # These can appear multiple times for the same key (matched via MULTILINE) + chunks = re.findall(r'^\s*([\w\.]+)=SEGMENT_START (.*?) SEGMENT_END', paste_data, re.MULTILINE | re.DOTALL) + for key, val in chunks: + reassembled[key] = reassembled.get(key, '') + val + + # 2. Handle simple fields: {key}={content}###END### + # We use a negative lookahead to skip keys handled by the chunked regex + simples = re.findall(r'^\s*([\w\.]+)=(?!SEGMENT_START)(.*?)###END###', paste_data, re.MULTILINE | re.DOTALL) + for key, val in simples: + # Only add if not already reassembled (prioritize chunks) + if key not in reassembled: + reassembled[key] = val + + # Return as ||| joined pairs for _parse_overrides + return '|||'.join([f"{k}={v}" for k, v in reassembled.items()]) + + def generate_now_application_sql(now_number): + def sql_chunk(field_sql, alias): + # Split the JSON field into 2000-char segments with robust start/end markers for reassembly. + return f"(SELECT string_agg(' {alias}=SEGMENT_START ' || substring(t.data from i for 2000) || ' SEGMENT_END ', E'\\n') || '###END###' FROM (SELECT COALESCE(({field_sql})::text, '[]') as data) t, generate_series(1, length(t.data), 2000) i)" + return ( + "SELECT 'now_application_guid=' || i.now_application_guid || '###END###' || E'\\n' || " + "'now_number=' || i.now_number || '###END###' || E'\\n' || " + + sql_chunk("SELECT json_agg(json_build_object('mine_party_appt_type_code', npa.mine_party_appt_type_code, 'party_name', p.party_name, 'first_name', p.first_name, 'email', p.email, 'phone_no', p.phone_no, 'has_signature', (p.signature IS NOT NULL), 'address', (SELECT json_build_object('address_line_1', address_line_1, 'city', city, 'sub_division_code', sub_division_code, 'post_code', post_code) FROM address WHERE party_guid = p.party_guid LIMIT 1)))::text FROM now_party_appointment npa JOIN party p ON npa.party_guid = p.party_guid WHERE npa.now_application_id = a.now_application_id AND npa.deleted_ind = false", "contacts_data") + " || E'\\n' || " + "'now_application.type_of_application=' || COALESCE(a.type_of_application, '') || '###END###' || E'\\n' || " + "'now_application.now_application_status_code=' || COALESCE(a.now_application_status_code, '') || '###END###' || E'\\n' || " + "'now_application.notice_of_work_type_code=' || COALESCE(a.notice_of_work_type_code, '') || '###END###' || E'\\n' || " + "'now_application.property_name=' || REPLACE(COALESCE(a.property_name, ''), ',', ' ') || '###END###' || E'\\n' || " + "'now_application.tenure_number=' || COALESCE(a.tenure_number, '') || '###END###' || E'\\n' || " + "'now_application.latitude=' || COALESCE(a.latitude::text, '') || '###END###' || E'\\n' || " + "'now_application.longitude=' || COALESCE(a.longitude::text, '') || '###END###' || E'\\n' || " + "'now_application.submitted_date=' || COALESCE(a.submitted_date::text, '') || '###END###' || E'\\n' || " + "'now_application.received_date=' || COALESCE(a.received_date::text, '') || '###END###' || E'\\n' || " + "'now_application.proposed_start_date=' || COALESCE(a.proposed_start_date::text, '') || '###END###' || E'\\n' || " + "'now_application.proposed_end_date=' || COALESCE(a.proposed_end_date::text, '') || '###END###' || E'\\n' || " + "'now_application.proponent_submitted_permit_number=' || COALESCE(a.proponent_submitted_permit_number, '') || '###END###' || E'\\n' || " + "'now_application.annual_summary_submitted=' || COALESCE(a.annual_summary_submitted::text, '') || '###END###' || E'\\n' || " + "'now_application.is_first_year_of_multi=' || COALESCE(a.is_first_year_of_multi::text, '') || '###END###' || E'\\n' || " + "'now_application.application_permit_type_code=' || COALESCE(a.application_permit_type_code, '') || '###END###' || E'\\n' || " + "'site_property_tenure_type_code=' || COALESCE((SELECT mine_tenure_type_code FROM mine_type WHERE now_application_guid = i.now_application_guid AND active_ind = true LIMIT 1), '') || '###END###' || E'\\n' || " + "'issuing_inspector_name=' || COALESCE((SELECT p.first_name || ' ' || p.party_name FROM party p WHERE p.party_guid = a.issuing_inspector_party_guid), '') || '###END###' || E'\\n' || " + + sql_chunk("SELECT json_build_object('has_community_water_shed', has_community_water_shed, 'has_archaeology_sites_affected', has_archaeology_sites_affected, 'authorization_details', authorization_details, 'has_licence_of_occupation', has_licence_of_occupation, 'licence_of_occupation', licence_of_occupation)::text FROM state_of_land WHERE now_application_id = a.now_application_id", "state_of_land_data") + " || E'\\n' || " + + sql_chunk("SELECT json_build_object('has_storage_explosive_on_site', has_storage_explosive_on_site, 'explosive_permit_issued', explosive_permit_issued, 'explosive_permit_number', explosive_permit_number)::text FROM blasting_operation WHERE now_application_id = a.now_application_id", "blasting_operation_data") + " || E'\\n' || " + + sql_chunk("SELECT json_agg(json_build_object('application_progress_status_code', application_progress_status_code, 'start_date', start_date::text, 'end_date', end_date::text, 'created_by', created_by, 'active_ind', active_ind))::text FROM now_application_progress WHERE now_application_id = a.now_application_id", "now_application_progress_data") + " || E'\\n' || " + + sql_chunk("SELECT json_agg(json_build_object('now_application_review_type_code', now_application_review_type_code, 'response_date', response_date::text, 'referee_name', REPLACE(COALESCE(referee_name, ''), ',', ' '), 'referral_number', REPLACE(COALESCE(referral_number, ''), ',', ' '), 'response_url', REPLACE(COALESCE(response_url, ''), ',', ' ')))::text FROM now_application_review WHERE now_application_id = a.now_application_id", "now_application_review_data") + " || E'\\n' || " + + sql_chunk("SELECT json_agg(json_build_object('now_application_document_type_code', now_application_document_type_code, 'is_final_package', is_final_package))::text FROM now_application_document_xref WHERE now_application_id = a.now_application_id AND now_application_document_type_code IN ('MRP', 'ACP') AND is_final_package = true AND deleted_ind = false", "documents_data") + " || E'\\n' || " + + sql_chunk("SELECT json_agg(json_build_object('permit_amendment_status_code', pa.permit_amendment_status_code, 'permit_amendment_type_code', pa.permit_amendment_type_code, 'received_date', pa.received_date::text, 'issue_date', pa.issue_date::text, 'authorization_end_date', pa.authorization_end_date::text, 'description', pa.description, 'permit_no', p.permit_no, 'permit_status_code', p.permit_status_code))::text FROM permit_amendment pa JOIN permit p ON pa.permit_id = p.permit_id WHERE pa.now_application_guid = i.now_application_guid AND pa.deleted_ind = false", "permit_amendments_data") + " " + "FROM now_application a JOIN now_application_identity i ON a.now_application_id = i.now_application_id " + f"WHERE i.now_number = '{now_number}';" + ) + + @app.cli.command('generate-now-query') + @click.option('--now-number', required=True, help='The NoW Number (e.g. 0400022-2025-01)') + def generate_now_query(now_number): + import logging + logging.getLogger().setLevel(logging.CRITICAL) + import sys + sys.stdout.write(generate_now_application_sql(now_number)) + + @app.cli.command('create-test-data') + @click.option('--scenario', type=click.Choice(['now-application']), default='now-application', help='The data scenario to create.') + @click.option('--name', help='Mine name override.') + @click.option('--mine-guid', help='Existing Mine GUID to link the data to.') + @click.option('--num', default=1, help='Number of entities to create (if applicable).') + @click.option('--override', multiple=True, help='Field overrides in key=value format (e.g. --override mine_name="New Name")') + @click.option('--interactive/--non-interactive', default=True, help='Run in interactive mode if no arguments provided.') + def create_test_data_command(scenario, name, mine_guid, num, override, interactive): + """ + Creates test data for a given scenario. Restricted to non-production environments. + """ + if Config.ENVIRONMENT_NAME not in ['local', 'dev', 'test']: + click.echo(f"Error: Command not allowed in {Config.ENVIRONMENT_NAME} environment.", err=True) + return + + User._test_mode = True + + # Check for piped input from STDIN (e.g. from oc exec or cat) + import sys + if not sys.stdin.isatty(): + # Read all available input + piped_data = sys.stdin.read().strip() + if piped_data: + processed_data = process_override_input(piped_data) + override = list(override) + [processed_data] + + # If no core arguments provided, force interactive mode + if interactive: + click.echo("--- MDS Test Data Generator (Interactive Mode) ---") + + if not scenario: + scenario = 'now-application' + + if scenario == 'now-application': + click.echo("\n[Production Data Helper]") + now_num_for_sql = click.prompt("Enter the NoW Number you want to replicate (e.g. 0400022-2025-01)", default="YOUR_NOW_NUMBER") + click.echo("\nTip: Run this SQL in the Prod Read-Only DB to get the override string.") + click.echo("--------------------------------------------------------------------------------") + + sql = generate_now_application_sql(now_num_for_sql) + click.echo(sql) + click.echo("--------------------------------------------------------------------------------") + + click.echo("Paste the override string, OR the path to a .txt file containing it.") + click.echo("If pasting, type 'DONE' on a new line or Ctrl+D to finish.") + import sys + paste_lines = [] + while True: + try: + line = sys.stdin.readline() + if not line: # EOF + break + if line.strip() == "DONE": + break + paste_lines.append(line) + except EOFError: + break + + paste_data = "".join(paste_lines).strip() + + if paste_data: + data_to_add = process_override_input(paste_data) + override = list(override) + [data_to_add] + # Handle file:// prefix often added by file explorers when copying paths + clean_path = paste_data.replace('file://', '') + clean_path = os.path.expanduser(clean_path) + + # Inside Docker, the repo root is often /app + # Check absolute path, then check relative to /app (repo root) + possible_paths = [clean_path] + if not clean_path.startswith('/app/'): + relative_path = os.path.join('/app', clean_path.lstrip('/')) + possible_paths.append(relative_path) + + found_file = False + # Only check for file if the paste_data looks like a single line path + if "\n" not in paste_data and len(paste_data) < 1000: + for p in possible_paths: + if os.path.isfile(p): + with open(p, 'r') as f: + data_to_add = f.read().strip() + click.echo(f" - SUCCESS: Loaded {len(data_to_add)} characters from {p}") + found_file = True + break + + # If loaded from file, apply same sanitization + if found_file: + if data_to_add.startswith('"') and data_to_add.endswith('"'): + data_to_add = data_to_add[1:-1] + data_to_add = data_to_add.replace('\\n', '\n') + data_to_add = data_to_add.replace('\\r', '\r') + data_to_add = data_to_add.replace('""', '"') + + if not found_file and any(p.endswith(('.txt', '.json')) for p in possible_paths) and "\n" not in paste_data: + click.secho(f" - WARNING: Treated input as raw string because no file was found at: {', '.join(possible_paths)}", fg='yellow') + click.secho(f" (Note: Since this runs in Docker, files must be inside the repo folder to be visible)", fg='yellow') + elif not found_file and len(paste_data) >= 4095 and "\n" not in paste_data: + click.secho(" - WARNING: Input is exactly 4095 characters. This common terminal limit often truncates pasted data.", fg='red', bold=True) + + override = list(override) + [data_to_add] + + + if not mine_guid: + mine_guid = click.prompt("Existing Mine GUID (the mine you want this record attached to)", default="") + + + with app.app_context(): + # Reset sequences to avoid IntegrityErrors in persistent environments + _reset_factory_sequences() + + try: + mine = None + if mine_guid: + mine = Mine.find_by_mine_guid(mine_guid) + if not mine: + click.echo(f"Error: Mine with GUID {mine_guid} not found.", err=True) + return + click.echo(f"Using existing Mine: {mine.mine_name} ({mine.mine_guid})") + + overrides = _parse_overrides(override) + + if scenario == 'now-application': + from datetime import datetime + from app.api.parties.party.models.party import Party + from app.api.parties.party.models.address import Address + from app.api.now_applications.models.now_party_appointment import NOWPartyAppointment + from app.api.now_applications.models.now_application_identity import NOWApplicationIdentity + from app.api.now_applications.models.now_application_progress import NOWApplicationProgress + from app.api.now_applications.models.now_application_review import NOWApplicationReview + from app.api.now_applications.models.now_application_delay import NOWApplicationDelay + from app.api.mines.mine.models.mine_type import MineType + from app.api.now_applications.models.now_application_document_xref import NOWApplicationDocumentXref + from app.api.mines.documents.models.mine_document import MineDocument + + mine_kwargs = {} + if not mine: + mine_kwargs = {'mine_name': name} if name else {} + m_kwargs = {k.replace('mine__', ''): v for k, v in overrides.items() if k.startswith('mine__')} + mine_kwargs.update(m_kwargs) + mine = MineFactory(**mine_kwargs) + + # Extract related data before passing to main factory + progress_data = overrides.pop('now_application_progress_data', None) + review_data = overrides.pop('now_application_review_data', None) + state_of_land_data = overrides.pop('state_of_land_data', None) + blasting_data = overrides.pop('blasting_operation_data', None) + contacts_data = overrides.pop('contacts_data', None) + documents_data = overrides.pop('documents_data', None) + inspector_name = overrides.pop('issuing_inspector_name', None) + tenure_type_code = overrides.pop('site_property_tenure_type_code', None) + permit_amendments_data = overrides.pop('permit_amendments_data', None) + + now_kwargs = {'mine': mine} + now_kwargs.update(overrides) + + # Extract submitted permit number for potential linkage + submitted_permit_no = now_kwargs.get('proponent_submitted_permit_number') + local_permit = None + if submitted_permit_no and mine: + local_permit = Permit.find_by_permit_no(submitted_permit_no) + if local_permit: + click.echo(f" - DEBUG: Found matching local permit {submitted_permit_no} for role linkage.") + + app_overrides = {} + for k in list(now_kwargs.keys()): + if k.startswith('now_application.') or k.startswith('now_application__'): + key_part = k.replace('now_application.', '').replace('now_application__', '') + app_overrides[key_part] = now_kwargs.pop(k) + + if k.startswith('mine__'): + del now_kwargs[k] + + + # Check for existing application to prompt overwrite + now_num = now_kwargs.get('now_number') + if now_num: + existing_identity = NOWApplicationIdentity.find_by_now_number(now_num) + if existing_identity: + if interactive: + if not click.confirm(f"Application {now_num} already exists. Would you like to OVERWRITE it?", abort=True): + return + click.echo(f"Overwriting application {now_num}...") + from sqlalchemy import text + nid = existing_identity.now_application_id + nguid = str(existing_identity.now_application_guid) + + # Use raw SQL to bypass SQLAlchemy's complex relationship/PK management for clean deletion + if nid: + # 1. Delete Activity Sub-children (Details and Xrefs) + db.session.execute(text("DELETE FROM activity_equipment_xref WHERE activity_summary_id IN (SELECT activity_summary_id FROM activity_summary WHERE now_application_id = :id)"), {"id": nid}) + db.session.execute(text("DELETE FROM activity_summary_detail_xref WHERE activity_summary_id IN (SELECT activity_summary_id FROM activity_summary WHERE now_application_id = :id)"), {"id": nid}) + db.session.execute(text("DELETE FROM activity_summary_staging_area_detail_xref WHERE activity_summary_id IN (SELECT activity_summary_id FROM activity_summary WHERE now_application_id = :id)"), {"id": nid}) + db.session.execute(text("DELETE FROM activity_summary_building_detail_xref WHERE activity_summary_id IN (SELECT activity_summary_id FROM activity_summary WHERE now_application_id = :id)"), {"id": nid}) + + # 2. Delete Activity Specific (Polymorphic Children) - ONLY those with their own tables (Joined Inheritance) + activity_tables = [ + 'camp', 'placer_operation', 'settling_pond', 'surface_bulk_sample', + 'sand_gravel_quarry_operation', 'underground_exploration', + 'exploration_access', 'exploration_surface_drilling' + ] + for table in activity_tables: + db.session.execute(text(f"DELETE FROM {table} WHERE activity_summary_id IN (SELECT activity_summary_id FROM activity_summary WHERE now_application_id = :id)"), {"id": nid}) + + # 3. Delete Activity Summaries + db.session.execute(text("DELETE FROM activity_summary WHERE now_application_id = :id"), {"id": nid}) + + # 4. Delete Direct Children of NOWApplication + db.session.execute(text("DELETE FROM blasting_operation WHERE now_application_id = :id"), {"id": nid}) + db.session.execute(text("DELETE FROM state_of_land WHERE now_application_id = :id"), {"id": nid}) + db.session.execute(text("DELETE FROM now_application_review WHERE now_application_id = :id"), {"id": nid}) + db.session.execute(text("DELETE FROM now_application_progress WHERE now_application_id = :id"), {"id": nid}) + db.session.execute(text("DELETE FROM now_party_appointment WHERE now_application_id = :id"), {"id": nid}) + db.session.execute(text("DELETE FROM now_application_document_xref WHERE now_application_id = :id"), {"id": nid}) + db.session.execute(text("DELETE FROM application_reason_code_xref WHERE now_application_id = :id"), {"id": nid}) + db.session.execute(text("DELETE FROM now_application_document_identity_xref WHERE now_application_id = :id"), {"id": nid}) + # Delete linked permit amendments + db.session.execute(text("DELETE FROM permit_amendment WHERE now_application_guid = :guid"), {"guid": nguid}) + + # 5. Clean up Identity-linked records + db.session.execute(text("DELETE FROM now_application_delay WHERE now_application_guid = :guid"), {"guid": nguid}) + db.session.execute(text("DELETE FROM mine_type WHERE now_application_guid = :guid"), {"guid": nguid}) + + # 6. Delete Identity itself (Must happen before NOWApplication because it references it) + db.session.execute(text("DELETE FROM now_application_identity WHERE now_application_guid = :guid"), {"guid": nguid}) + + # 7. Finally delete the application itself + if nid: + db.session.execute(text("DELETE FROM now_application WHERE now_application_id = :id"), {"id": nid}) + + db.session.commit() + click.echo(f"Existing record deleted successfully. Recreating...") + + # Create NOWApplication + now_identity = NOWApplicationIdentityFactory(**now_kwargs) + now_app = now_identity.now_application + db.session.flush() + + # Apply application-level overrides + for k, v in app_overrides.items(): + if hasattr(now_app, k): + setattr(now_app, k, v) + + # Replicate Inspector if provided + if inspector_name: + name_parts = inspector_name.split() + last_name = name_parts[-1] if name_parts else 'Inspector' + first_name = name_parts[0] if len(name_parts) > 1 else 'Lead' + + inspector = Party.find_by_name(last_name, first_name) + if not inspector: + inspector = Party.create(last_name, "604-555-1212", "PER", first_name=first_name) + db.session.flush() + + if not inspector.signature: + inspector.signature = DUMMY_SIGNATURE + db.session.add(inspector) + db.session.flush() + + now_app.issuing_inspector_party_guid = inspector.party_guid + + # Replicate Site Property (MineType) + if tenure_type_code: + MineType.create_or_update_mine_type_with_details( + mine_guid=mine.mine_guid, + now_application_guid=now_identity.now_application_guid, + mine_tenure_type_code=tenure_type_code + ) + + # Update StateOfLand and BlastingOperation + if state_of_land_data and isinstance(state_of_land_data, dict): + for k, v in state_of_land_data.items(): + if hasattr(now_app.state_of_land, k): setattr(now_app.state_of_land, k, v) + + if blasting_data and isinstance(blasting_data, dict): + for k, v in blasting_data.items(): + if hasattr(now_app.blasting_operation, k): setattr(now_app.blasting_operation, k, v) + + def _parse_date(d_str): + if not d_str or d_str == 'None': return None + try: + # Split to handle both 'YYYY-MM-DD' and 'YYYY-MM-DD HH:MM:SS+00' + return datetime.strptime(d_str.split(' ')[0], '%Y-%m-%d').date() + except: return None + + # Replicate Progress records + if progress_data and isinstance(progress_data, list): + from app.api.now_applications.models.now_application_progress import NOWApplicationProgress + db.session.query(NOWApplicationProgress).filter_by(now_application_id=now_app.now_application_id).delete() + for p in progress_data: + p['start_date'] = _parse_date(p.get('start_date')) + p['end_date'] = _parse_date(p.get('end_date')) + NOWApplicationProgressFactory(now_application=now_app, **p) + + # Replicate Review records + if review_data and isinstance(review_data, list): + from app.api.now_applications.models.now_application_review import NOWApplicationReview + db.session.query(NOWApplicationReview).filter_by(now_application_id=now_app.now_application_id).delete() + for r in review_data: + r['response_date'] = _parse_date(r.get('response_date')) + NOWApplicationReviewFactory(now_application=now_app, **r) + + last_processed_permit = local_permit + if permit_amendments_data and isinstance(permit_amendments_data, list): + click.echo(f" - DEBUG: Ingesting {len(permit_amendments_data)} permit fragments") + for pa_doc in permit_amendments_data: + click.echo(f" - Processing '{pa_doc['permit_no']}' / Status: {pa_doc['permit_amendment_status_code']}") + + # 1. Fetch or Create Parent Permit + permit = Permit.find_by_permit_no(pa_doc['permit_no']) + if not permit: + permit = Permit.create( + mine=mine, + permit_no=pa_doc['permit_no'], + permit_status_code=pa_doc['permit_status_code'], + is_exploration=False, + exemption_fee_status_code=None, + exemption_fee_status_note=None + ) + if permit._mine_associations: + permit._mine_associations[0].start_date = datetime.utcnow() + db.session.flush() + click.echo(f" - Created matching permit {pa_doc['permit_no']}") + else: + if mine.mine_guid not in [m.mine_guid for m in permit._all_mines]: + from app.api.mines.permits.permit.models.mine_permit_xref import MinePermitXref + permit._mine_associations.append(MinePermitXref(mine_guid=mine.mine_guid, start_date=datetime.utcnow())) + db.session.flush() + click.echo(f" - Added mine association for existing permit {pa_doc['permit_no']}") + + last_processed_permit = permit + + # 2. Check for existing amendment for this NoW on this permit + existing_pa = PermitAmendment.query.filter_by( + now_application_guid=now_identity.now_application_guid, + permit_id=permit.permit_id, + deleted_ind=False + ).first() + + if not existing_pa: + pa_guid = uuid.UUID(str(now_identity.now_application_guid)) + status = pa_doc['permit_amendment_status_code'] + + new_pa = PermitAmendment.create( + permit=permit, + mine=mine, + received_date=_parse_date(pa_doc['received_date']), + issue_date=_parse_date(pa_doc['issue_date']), + authorization_end_date=_parse_date(pa_doc['authorization_end_date']), + permit_amendment_type_code=pa_doc['permit_amendment_type_code'], + description=pa_doc['description'], + permit_amendment_status_code=status, + now_application_guid=pa_guid, + add_to_session=True + ) + click.echo(f" - Linked {status} amendment for permit {pa_doc['permit_no']}") + else: + click.echo(f" - Amendment for {pa_doc['permit_no']} already exists.") + + # Replicate Documents (MRP/ACP metadata) + if documents_data and isinstance(documents_data, list): + for doc in documents_data: + mine_doc = MineDocument(mine_guid=mine.mine_guid, document_name=f"Replicated_{doc['now_application_document_type_code']}.pdf", document_manager_guid=uuid.uuid4()) + db.session.add(mine_doc) + db.session.flush() + db.session.add(NOWApplicationDocumentXref(now_application_id=now_app.now_application_id, mine_document_guid=mine_doc.mine_document_guid, now_application_document_type_code=doc['now_application_document_type_code'], is_final_package=True)) + + # Replicate Contacts (Permittees) + # Handled AFTER permits to ensure we have a permit_id to link to for PMT role + if contacts_data: + if isinstance(contacts_data, list): + from app.api.parties.party_appt.models.mine_party_appt import MinePartyAppointment + + for c in contacts_data: + p_name = c.pop('party_name', '') + f_name = c.pop('first_name', '') + has_sig = c.pop('has_signature', False) + address_data = c.pop('address', None) + type_code = c.get('mine_party_appt_type_code', 'PMT') + + party = Party.find_by_name(p_name, f_name) + if not party: + mock_email = f"{f_name or 'info'}.{p_name}@example.com".replace(' ', '.').lower() + mock_phone = "604-555-0000" + party = Party.create(p_name, mock_phone, "PER" if f_name else "ORG", first_name=f_name, email=mock_email) + db.session.flush() + + if address_data: + addr = Address(party_guid=party.party_guid, **address_data) + db.session.add(addr) + db.session.flush() + + if has_sig and not party.signature: + party.signature = DUMMY_SIGNATURE + db.session.add(party) + + npa = NOWPartyAppointment(now_application_id=now_app.now_application_id, party_guid=party.party_guid, mine_party_appt_type_code=type_code) + db.session.add(npa) + click.echo(f" - DEBUG: Linked contact {p_name} as {type_code}") + + if type_code in PERMIT_LINKED_CONTACT_TYPES or type_code == 'MMG': + # Link to permit if it's a permit-linked type + is_permit_linked = type_code in PERMIT_LINKED_CONTACT_TYPES + target_permit_id = last_processed_permit.permit_id if (is_permit_linked and last_processed_permit) else None + + if is_permit_linked and not target_permit_id: + click.echo(f" - WARNING: Skipping MinePartyAppointment for {type_code} {p_name} because no permit_id found.") + continue + + # Retire existing active appointments for this ROLE on this MINE/PERMIT to avoid date overlap + query = MinePartyAppointment.query.filter_by( + mine_party_appt_type_code=type_code, + deleted_ind=False + ).filter(MinePartyAppointment.end_date == None) + + if is_permit_linked: + query = query.filter_by(permit_id=target_permit_id) + else: + query = query.filter_by(mine_guid=mine.mine_guid) + + existing_active = query.all() + for ea in existing_active: + ea.end_date = datetime.utcnow().date() + db.session.add(ea) + db.session.flush() + + db.session.add(MinePartyAppointment( + mine_guid=mine.mine_guid if not is_permit_linked else None, + party_guid=party.party_guid, + mine_party_appt_type_code=type_code, + permit_id=target_permit_id, + start_date=datetime.utcnow().date() + )) + db.session.flush() + click.echo(f" - Replicated {len(contacts_data)} Contacts") + else: + click.echo(f"Warning: contacts_data was not a list (got {type(contacts_data).__name__}). Skipping contact replication.", err=True) + + click.echo(f"Created/Linked Mine: {mine.mine_name} [GUID: {mine.mine_guid}]") + click.echo(f"Created NoW Application: {now_identity.now_number} [ID: {now_identity.now_application_id}] [GUID: {now_identity.now_application_guid}]") + + if progress_data is not None: + click.echo(f" - Replicated {len(progress_data) if isinstance(progress_data, list) else 0} Progress records") + if review_data is not None: + count = len(review_data) if isinstance(review_data, list) else 0 + click.echo(f" - Replicated {count} Review records") + if documents_data: + click.echo(f" - Replicated {len(documents_data)} Final Package document placeholders") + if permit_amendments_data: + count = len(permit_amendments_data) if isinstance(permit_amendments_data, list) else 0 + click.echo(f" - Replicated {count} Permit Amendment fragments") + + elif scenario == 'random-mine': + for _ in range(num): + mine_kwargs = {'mine_name': name} if name else {} + mine_kwargs.update(overrides) + new_mine = MineFactory(**mine_kwargs) + db.session.add(new_mine) + click.echo(f"Created Random Mine: {new_mine.mine_name} [GUID: {new_mine.mine_guid}]") + + db.session.commit() + click.echo("Success: Test data committed to database.") + except Exception as e: + db.session.rollback() + click.echo(f"Error: Failed to create test data. {e}", err=True) + raise e + + def _reset_factory_sequences(): + """ + Queries the database for the maximum IDs and resets factory-boy sequences + to prevent IntegrityError (UniqueViolation) in environments with existing data. + """ + from sqlalchemy import func + from app.api.now_submissions.models.client import Client as NOWClient + from app.api.now_submissions.models.application import Application as NOWApplication + from app.api.now_submissions.models.placer_activity import PlacerActivity as NOWPlacerActivity + from app.api.now_submissions.models.settling_pond import SettlingPondSubmission as NOWSettlingPond + from app.api.now_applications.models.now_application_identity import NOWApplicationIdentity + + from tests.now_submission_factories import ( + NOWClientFactory, NOWSubmissionFactory, + NOWPlacerActivityFactory, NOWSettlingPondFactory, + NOWApplicationNDAFactory + ) + from tests.now_application_factories import NOWApplicationIdentityFactory + + # Client ID (now_submissions.client) + max_client_id = db.session.query(func.max(NOWClient.clientid)).scalar() or 1000 + NOWClientFactory.reset_sequence(max_client_id + 1) + + # Message ID (now_submissions.application) + max_message_id = db.session.query(func.max(NOWApplication.messageid)).scalar() or 1000 + NOWSubmissionFactory.reset_sequence(max_message_id + 1) + NOWApplicationNDAFactory.reset_sequence(max_message_id + 1) + + # Activity IDs + max_placer_id = db.session.query(func.max(NOWPlacerActivity.placeractivityid)).scalar() or 1000 + NOWPlacerActivityFactory.reset_sequence(max_placer_id + 1) + + max_pond_id = db.session.query(func.max(NOWSettlingPond.settlingpondid)).scalar() or 1000 + NOWSettlingPondFactory.reset_sequence(max_pond_id + 1) + + # NOW Application Identity + max_mms_cid = db.session.query(func.max(func.cast(NOWApplicationIdentity.mms_cid, db.Integer))).scalar() or 1000 + NOWApplicationIdentityFactory.reset_sequence(max_mms_cid + 1) + + click.echo(f"Factory sequences reset (Client: {max_client_id}, Msg: {max_message_id}, MMS: {max_mms_cid})") + + def split_top_level_commas(s): + """Split a string by commas or newlines, but only those at the top level and followed by a key=val part.""" + import re + parts = [] + current = [] + depth = 0 + in_quote = False + quote_char = None + + click.echo(f" - DEBUG Splitter: Processing string of length {len(s)}") + + for i, char in enumerate(s): + # Track quotes at any depth to avoid counting braces/commas inside strings + if char in '"\'' and (i == 0 or s[i-1] != '\\'): + if not in_quote: + in_quote = True + quote_char = char + elif char == quote_char: + in_quote = False + + if not in_quote: + if char in '[{': + depth += 1 + elif char in ']}': + depth -= 1 + elif char in (',', '\n') and depth == 0: + # Look ahead to see if the next part looks like "key=" + lookahead = s[i+1:].lstrip() + # Match key= or key.subkey= and allow for spaces + if re.match(r'^[\w\.]+\s*=', lookahead): + p = ''.join(current).strip() + if p: + # Handle potential truncation if the field starts with [ or { + # by ensuring we at least try to close it if it's the last part. + # Actually, split_top_level_commas is called BEFORE _parse_overrides. + parts.append(p) + current = [] + continue + + current.append(char) + + if current: + p = ''.join(current).strip() + if p: + parts.append(p) + + click.echo(f" - DEBUG Splitter: Split into {len(parts)} segments. Final depth={depth}, in_quote={in_quote}") + return parts + + def _parse_overrides(overrides): + import json + parsed = {} + clean_pairs = [] + + for o in overrides: + if '|||' in o: + clean_pairs.extend(o.split('|||')) + elif '--override' in o: + clean_pairs.extend([p.strip() for p in o.split('--override ') if p.strip()]) + else: + clean_pairs.extend(split_top_level_commas(o)) + + for pair in clean_pairs: + if '=' in pair: + key, value = pair.split('=', 1) + key = key.replace('.', '__').strip() + + # Debug: show raw value for contacts + if 'contacts' in key.lower(): + click.echo(f" - DEBUG: Raw pair for {key}, value length={len(value)}") + click.echo(f" - Value starts: '{value[:50]}'") + click.echo(f" - Value ends: '{value[-50:]}'") + + value = value.strip().strip('"\'') + + # Try JSON parsing - be more aggressive in detecting JSON + v_clean = value.strip() + if (v_clean.startswith('[') or v_clean.startswith('{')): + # If it looks like JSON but is missing the closing bracket/brace, + # it might have been truncated by the terminal. Try to fix it. + if v_clean.startswith('[') and not v_clean.endswith(']'): + v_clean += ']' + if v_clean.startswith('{') and not v_clean.endswith('}'): + v_clean += '}' + + try: + parsed[key] = json.loads(v_clean) + # click.echo(f" - DEBUG: Parsed JSON for {key}") # Keep quiet if success + continue + except json.JSONDecodeError as e: + # If still failing, it might be truncated in the middle of a string/object + # We try one more level of aggressive fixing for common list truncation + if v_clean.startswith('[') and '},' in v_clean: + try: + fixed_v = v_clean[:v_clean.rindex('}')+1] + ']' + parsed[key] = json.loads(fixed_v) + click.secho(f" - FIXED: Recovered truncated JSON for {key}", fg='green') + continue + except: pass + click.echo(f"Warning: JSON decode error for key {key}: {str(e)}", err=True) + click.echo(f" - Raw value starts: '{v_clean[:100]}'") + click.echo(f" - Raw value ends: '{v_clean[-100:]}'") + elif key == 'contacts_data': + click.echo(f" - DEBUG: contacts_data did NOT parse as JSON.") + click.echo(f" - Cleaned value starts with: '{v_clean[:20]}'") + click.echo(f" - Cleaned value ends with: '{v_clean[-20:]}'") + # If it's missing the final bracket, maybe it was split wrong + if '[' in v_clean and ']' not in v_clean: + click.echo(f" - WARNING: contacts_data appears truncated (has '[' but no ']')") + + v_lower = value.lower() + if v_lower in ('true', 't'): + parsed[key] = True + elif v_lower in ('false', 'f'): + parsed[key] = False + elif v_lower in ('none', 'null', ''): + parsed[key] = None + else: + try: + parsed[key] = int(value) + except ValueError: + try: + parsed[key] = float(value) + except ValueError: + parsed[key] = value + + if 'contacts_data' in parsed: + c_data = parsed['contacts_data'] + click.echo(f" - DEBUG: contacts_data found, type={type(c_data).__name__}, len={len(c_data)}") + else: + click.echo(f" - DEBUG: contacts_data NOT found in overrides. Keys present: {list(parsed.keys())}") + + return parsed + def _batch( items: list, batch_size: int) -> List[List]: """Split a list into batches of specified size"""